From 78c2e1c978e745e0a23a5f4cc61ca9b0c0152f7a Mon Sep 17 00:00:00 2001
From: Guy Veis <guyveis@gmail.com>
Date: Sun, 8 Mar 2015 15:20:58 +0200
Subject: [PATCH 0001/2145] Fixed parsing spaced xml files

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f9e8e2bad..1ce16a779 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -439,7 +439,7 @@ class InfoExtractor(object):
             return xml_string
         if transform_source:
             xml_string = transform_source(xml_string)
-        return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
+        return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8').strip())
 
     def _download_json(self, url_or_request, video_id,
                        note='Downloading JSON metadata',

From 8bc516c67be299e9288b0ebb31bda0d1dc4e4e19 Mon Sep 17 00:00:00 2001
From: Guy Veis <guyveis@gmail.com>
Date: Sun, 8 Mar 2015 15:24:30 +0200
Subject: [PATCH 0002/2145] Fixed walla extractor, added site-wide video
 extraction

---
 youtube_dl/extractor/walla.py | 63 +++++++++++++++++++++++++----------
 1 file changed, 46 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index 24efbd6e6..a1326df74 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -5,29 +5,43 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     xpath_text,
     int_or_none,
 )
 
 
 class WallaIE(InfoExtractor):
-    _VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
-    _TEST = {
+    _VALID_URL = r'http://[^\.]+\.walla\.co\.il/[^/]+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://news.walla.co.il/item/2835878',
+        'info_dict': {
+            'id': '2663876',
+            'ext': 'mp4',
+            'title': 'בנט יורה: "בפעם הבאה יהיה רב ראשי אחד לישראל"',
+            'description': 'md5:5f3ac43a8abc132ccaa1a6894a137440',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 112,
+        },
+        'params': {
+            # stream download
+            'skip_download': True,
+        }
+    }, {
         'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
         'info_dict': {
             'id': '2642630',
-            'display_id': 'one-direction-all-for-one',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'וואן דיירקשן: ההיסטריה',
             'description': 'md5:de9e2512a92442574cdb0913c49bc4d8',
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 3600,
         },
         'params': {
-            # rtmp download
+            # stream download
             'skip_download': True,
         }
-    }
+    }]
 
     _SUBTITLE_LANGS = {
         'עברית': 'heb',
@@ -36,18 +50,27 @@ class WallaIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
 
         video = self._download_xml(
-            'http://video2.walla.co.il/?w=null/null/%s/@@/video/flv_pl' % video_id,
-            display_id)
+            'http://video.walla.co.il/?w=//%s/@@/video/flv_pl' % video_id,
+            video_id)
 
         item = video.find('./items/item')
 
+        if item is None:
+            raise ExtractorError('The item doesn\'t exist or has no video.', expected=True)
+
         title = xpath_text(item, './title', 'title')
-        description = xpath_text(item, './synopsis', 'description')
+        description = next(
+            item for item in [
+                xpath_text(item, './synopsis', 'description'),
+                xpath_text(item, './subtitle', 'description'),
+                '',
+            ] if item is not None
+        )
         thumbnail = xpath_text(item, './preview_pic', 'thumbnail')
         duration = int_or_none(xpath_text(item, './duration', 'duration'))
+        default_file = xpath_text(item, './src', 'src')
 
         subtitles = {}
         for subtitle in item.findall('./subtitles/subtitle'):
@@ -57,16 +80,23 @@ class WallaIE(InfoExtractor):
                 'url': xpath_text(subtitle, './src'),
             }]
 
+        playlist_url = 'http://walla-s.vidnt.com/walla_vod/_definst_/%s.mp4/playlist.m3u8'
+
         formats = []
+        formats.append(
+            {
+                'url': playlist_url % default_file,
+                'ext': 'mp4',
+                'format_id': 40,
+            }
+        )
+
         for quality in item.findall('./qualities/quality'):
             format_id = xpath_text(quality, './title')
             fmt = {
-                'url': 'rtmp://wafla.walla.co.il/vod',
-                'play_path': xpath_text(quality, './src'),
-                'player_url': 'http://isc.walla.co.il/w9/swf/video_swf/vod/WallaMediaPlayerAvod.swf',
-                'page_url': url,
-                'ext': 'flv',
-                'format_id': xpath_text(quality, './title'),
+                'url': playlist_url % xpath_text(quality, './src'),
+                'ext': 'mp4',
+                'format_id': quality.attrib['type'],
             }
             m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
             if m:
@@ -76,7 +106,6 @@ class WallaIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 80a35f5321463457819eb20e1300dacfd9177a50 Mon Sep 17 00:00:00 2001
From: Guy Veis <guyveis@gmail.com>
Date: Sun, 8 Mar 2015 15:37:49 +0200
Subject: [PATCH 0003/2145] Fixed test item URL

---
 youtube_dl/extractor/walla.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index a1326df74..9345efb11 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -14,7 +14,7 @@ from ..utils import (
 class WallaIE(InfoExtractor):
     _VALID_URL = r'http://[^\.]+\.walla\.co\.il/[^/]+/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://news.walla.co.il/item/2835878',
+        'url': 'http://news.walla.co.il/item/2663876',
         'info_dict': {
             'id': '2663876',
             'ext': 'mp4',

From f11554092b419baa919875432fe6ebc1f22f5307 Mon Sep 17 00:00:00 2001
From: Tjark Saul <tjark.saul@wachplan.net>
Date: Fri, 17 Apr 2015 09:21:54 +0200
Subject: [PATCH 0004/2145] [Lecture2Go] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/lecture2go.py | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/lecture2go.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..3d6e981b2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -249,6 +249,7 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .lecture2go import Lecture2GoIE
 from .letv import (
     LetvIE,
     LetvTvIE,
diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
new file mode 100644
index 000000000..9cf28e31c
--- /dev/null
+++ b/youtube_dl/extractor/lecture2go.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Lecture2GoIE(InfoExtractor):
+    _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
+        'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
+        'info_dict': {
+            'id': '17473',
+            'ext': 'mp4',
+            'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4',
+            'title': '2 - Endliche Automaten und reguläre Sprachen'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<em class="title">(.*?)</em>', webpage, 'title')
+        video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
+        creator = self._html_search_regex(r'<div id="description">(.*)</div>', webpage, 'creator')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'creator': creator
+        }

From 233c1c0e76d64c9e13dc8968bfd8a014c49e66a8 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 3 May 2015 11:04:14 +0300
Subject: [PATCH 0005/2145] [downloader/f4m] Fragment filenames must be
 sanitized

because the fragment was written to a file with a sanitized name by
http_dl.download()
---
 youtube_dl/downloader/f4m.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b1a858c45..3cb07e15f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -396,18 +396,19 @@ class F4mFD(FileDownloader):
                 success = http_dl.download(frag_filename, {'url': url})
                 if not success:
                     return False
-                with open(frag_filename, 'rb') as down:
-                    down_data = down.read()
-                    reader = FlvReader(down_data)
-                    while True:
-                        _, box_type, box_data = reader.read_box_info()
-                        if box_type == b'mdat':
-                            dest_stream.write(box_data)
-                            break
+                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
+                down_data = down.read()
+                down.close()
+                reader = FlvReader(down_data)
+                while True:
+                    _, box_type, box_data = reader.read_box_info()
+                    if box_type == b'mdat':
+                        dest_stream.write(box_data)
+                        break
                 if live:
-                    os.remove(frag_filename)
+                    os.remove(encodeFilename(frag_sanitized))
                 else:
-                    frags_filenames.append(frag_filename)
+                    frags_filenames.append(frag_sanitized)
             except (compat_urllib_error.HTTPError, ) as err:
                 if live and (err.code == 404 or err.code == 410):
                     # We didn't keep up with the live window. Continue
@@ -430,7 +431,7 @@ class F4mFD(FileDownloader):
         elapsed = time.time() - start
         self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
-            os.remove(frag_file)
+            os.remove(encodeFilename(frag_file))
 
         fsize = os.path.getsize(encodeFilename(filename))
         self._hook_progress({

From ac6c358c2ab882427f74af47fe5df762dd348c20 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 02:08:47 +0800
Subject: [PATCH 0006/2145] [teamcoco] Fix extracting preload data again

---
 youtube_dl/extractor/teamcoco.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 2381676b4..95d58ddd0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import binascii
 import re
 
 from .common import InfoExtractor
@@ -9,6 +10,7 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
+from ..compat import compat_ord
 
 
 class TeamcocoIE(InfoExtractor):
@@ -66,7 +68,7 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        preload = None
+        data = preload = None
         preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
         if preloads:
             preload = max([(len(p), p) for p in preloads])[1]
@@ -80,11 +82,27 @@ class TeamcocoIE(InfoExtractor):
             ], webpage.replace('","', ''), 'preload data', default=None)
 
         if not preload:
+            preload_codes = self._html_search_regex(
+                r'(function.+)setTimeout\(function\(\)\{playlist',
+                webpage, 'preload codes')
+            base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+            base64_fragments.remove('init')
+            for i in range(len(base64_fragments)):
+                cur_sequence = (''.join(base64_fragments[i:] + base64_fragments[:i])).encode('ascii')
+                try:
+                    raw_data = base64.b64decode(cur_sequence)
+                except (TypeError, binascii.Error):
+                    continue
+                if compat_ord(raw_data[0]) == compat_ord('{'):
+                    data = self._parse_json(raw_data.decode('utf-8'), video_id, fatal=False)
+
+        if not preload and not data:
             raise ExtractorError(
                 'Preload information could not be extracted', expected=True)
 
-        data = self._parse_json(
-            base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
+        if not data:
+            data = self._parse_json(
+                base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From d9a743d9178b0ed1e44168e42e8cec2d7dd8d63e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:05:37 +0800
Subject: [PATCH 0007/2145] [vice] Remove a redundant print

---
 youtube_dl/extractor/vice.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 71f520fb5..04e2b0ba7 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -31,7 +31,6 @@ class ViceIE(InfoExtractor):
                 r'embedCode=([^&\'"]+)', webpage,
                 'ooyala embed code')
             ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
-            print(ooyala_url)
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
         return self.url_result(ooyala_url, ie='Ooyala')

From 05d5392cdaa558dba285c328182d4f3e82fb8e8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:06:22 +0800
Subject: [PATCH 0008/2145] [common] Ignore subtitles in m3u8

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 34e27a35a..981e34bc7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -896,7 +896,7 @@ class InfoExtractor(object):
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                last_media_name = last_media.get('NAME') if last_media else None
+                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
                 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),

From 84bf31aaf8b9b7397de5f3189295d93e8e93e5e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:12:01 +0800
Subject: [PATCH 0009/2145] [ooyala] Extract m3u8 information (#2292)

---
 youtube_dl/extractor/ooyala.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index d5b05c18f..b33e8230d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     unescapeHTML,
     ExtractorError,
+    determine_ext,
 )
 
 
@@ -44,11 +45,21 @@ class OoyalaIE(InfoExtractor):
                               ie=cls.ie_key())
 
     def _extract_result(self, info, more_info):
+        embedCode = info['embedCode']
+        video_url = info.get('ipad_url') or info['url']
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': 'mp4',
+            }]
+
         return {
-            'id': info['embedCode'],
-            'ext': 'mp4',
+            'id': embedCode,
             'title': unescapeHTML(info['title']),
-            'url': info.get('ipad_url') or info['url'],
+            'formats': formats,
             'description': unescapeHTML(more_info['description']),
             'thumbnail': more_info['promo'],
         }

From c09593c04e0b345df02cae663dc064d29e241cba Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 15:07:11 +0100
Subject: [PATCH 0010/2145] [BildIE] Escape ampersands in xml and update test
 thumbnail

---
 youtube_dl/extractor/bild.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 77b562d99..ba0c185eb 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+  int_or_none,
+  fix_xml_ampersands,
+)
 
 
 class BildIE(InfoExtractor):
@@ -15,7 +18,7 @@ class BildIE(InfoExtractor):
             'id': '38184146',
             'ext': 'mp4',
             'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
+            'thumbnail': 'http://bilder.bild.de/fotos/bild-hat-sie-getestet-das-koennen-apples-neue-ipads-38184138/Bild/1.bild.jpg',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }
@@ -25,7 +28,7 @@ class BildIE(InfoExtractor):
         video_id = self._match_id(url)
 
         xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
-        doc = self._download_xml(xml_url, video_id)
+        doc = self._download_xml(xml_url, video_id, transform_source=fix_xml_ampersands)
 
         duration = int_or_none(doc.attrib.get('duration'), scale=1000)
 

From aafe2739909882931f7f624e83fe532af0bfafc1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 22:07:32 +0800
Subject: [PATCH 0011/2145] [ooyala] Use SAS API to extract info (fixes #4336)

---
 youtube_dl/extractor/ooyala.py | 43 ++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index b33e8230d..0b049274a 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -1,12 +1,14 @@
 from __future__ import unicode_literals
 import re
 import json
+import base64
 
 from .common import InfoExtractor
 from ..utils import (
     unescapeHTML,
     ExtractorError,
     determine_ext,
+    int_or_none,
 )
 
 
@@ -33,6 +35,17 @@ class OoyalaIE(InfoExtractor):
                 'description': '',
             },
         },
+        {
+            # Information available only through SAS api
+            # From http://community.plm.automation.siemens.com/t5/News-NX-Manufacturing/Tool-Path-Divide/ba-p/4187
+            'url': 'http://player.ooyala.com/player.js?embedCode=FiOG81ZTrvckcchQxmalf4aQj590qTEx',
+            'md5': 'a84001441b35ea492bc03736e59e7935',
+            'info_dict': {
+                'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
+                'ext': 'mp4',
+                'title': 'Ooyala video',
+            }
+        }
     ]
 
     @staticmethod
@@ -88,6 +101,36 @@ class OoyalaIE(InfoExtractor):
                 mobile_player, 'info', fatal=False, default=None)
             if videos_info:
                 break
+
+        if not videos_info:
+            formats = []
+            auth_data = self._download_json(
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                embedCode)
+
+            cur_auth_data = auth_data['authorization_data'][embedCode]
+
+            for stream in cur_auth_data['streams']:
+                formats.append({
+                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
+                    'ext': stream.get('delivery_type'),
+                    'format': stream.get('video_codec'),
+                    'format_id': stream.get('profile'),
+                    'width': int_or_none(stream.get('width')),
+                    'height': int_or_none(stream.get('height')),
+                    'abr': int_or_none(stream.get('audio_bitrate')),
+                    'vbr': int_or_none(stream.get('video_bitrate')),
+                })
+            if len(formats):
+                return {
+                    'id': embedCode,
+                    'formats': formats,
+                    'title': 'Ooyala video',
+                }
+
+            if not cur_auth_data['authorized']:
+                raise ExtractorError(cur_auth_data['message'], expected=True)
+
         if not videos_info:
             raise ExtractorError('Unable to extract info')
         videos_info = videos_info.replace('\\"', '"')

From bc08873cff6d36ba175e5121b0ba1ad270c664c9 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 15:09:27 +0100
Subject: [PATCH 0012/2145] Fix indents

---
 youtube_dl/extractor/bild.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index ba0c185eb..41dd1cbc1 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-  int_or_none,
-  fix_xml_ampersands,
+    int_or_none,
+    fix_xml_ampersands,
 )
 
 

From f22834a3723dd59f1e04c27d0bc3a373ebc17183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 20:20:43 +0600
Subject: [PATCH 0013/2145] [bild] Relax thumbnail test check

---
 youtube_dl/extractor/bild.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 41dd1cbc1..4d8cce1ef 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -18,7 +18,7 @@ class BildIE(InfoExtractor):
             'id': '38184146',
             'ext': 'mp4',
             'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/bild-hat-sie-getestet-das-koennen-apples-neue-ipads-38184138/Bild/1.bild.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }

From 3799834dcff27ffeea66e3ec96166f8da8fa73ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 20:46:11 +0600
Subject: [PATCH 0014/2145] [YoutubeDL] Do not force bestvideo+bestaudio when
 outtmpl is stdout (#5627)

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb7470f72..d8583a8eb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,7 +1086,9 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if info_dict['extractor'] in ['youtube', 'ted'] and FFmpegMergerPP(self).available:
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
+                    and info_dict['extractor'] in ['youtube', 'ted']
+                    and FFmpegMergerPP(self).available):
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 406224be5231e602b543579706ad6056b75fbe68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 21:02:59 +0600
Subject: [PATCH 0015/2145] [extractor/generic] Fix following incomplete
 redirects (#5640)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d09e85665..cd7c47d6d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1453,7 +1453,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = found.group(1)
+                new_url = compat_urlparse.urljoin(url, found.group(1))
                 self.report_following_redirect(new_url)
                 return {
                     '_type': 'url',

From 5268a05e4722d74e125a97b023d92943745bb249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 May 2015 17:04:15 +0200
Subject: [PATCH 0016/2145] [ooyala] Style fix

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 0b049274a..c0e6d643d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -121,7 +121,7 @@ class OoyalaIE(InfoExtractor):
                     'abr': int_or_none(stream.get('audio_bitrate')),
                     'vbr': int_or_none(stream.get('video_bitrate')),
                 })
-            if len(formats):
+            if formats:
                 return {
                     'id': embedCode,
                     'formats': formats,

From 09b412dafa4bc95b9850d77b3bce5f7eac47a578 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 02:12:28 +0800
Subject: [PATCH 0017/2145] [nhl] Partial support for hlg id (fixes #4285)

---
 youtube_dl/extractor/nhl.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 407465998..b572370c2 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -21,6 +21,9 @@ class NHLBaseInfoExtractor(InfoExtractor):
         return json_string.replace('\\\'', '\'')
 
     def _real_extract_video(self, video_id):
+        vid_parts = video_id.split(',')
+        if len(vid_parts) == 3:
+            video_id = '%s0%s%s-X-h' % (vid_parts[0][:4], vid_parts[1], vid_parts[2].rjust(4, '0'))
         json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
         data = self._download_json(
             json_url, video_id, transform_source=self._fix_json)
@@ -60,7 +63,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P<id>[-0-9a-zA-Z,]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -101,6 +104,17 @@ class NHLIE(NHLBaseInfoExtractor):
     }, {
         'url': 'http://video.nhl.com/videocenter/?id=736722',
         'only_matching': True,
+    }, {
+        'url': 'http://video.nhl.com/videocenter/console?hlg=20142015,2,299&lang=en',
+        'md5': '076fcb88c255154aacbf0a7accc3f340',
+        'info_dict': {
+            'id': '2014020299-X-h',
+            'ext': 'mp4',
+            'title': 'Penguins at Islanders / Game Highlights',
+            'description': 'Home broadcast - Pittsburgh Penguins at New York Islanders - November 22, 2014',
+            'duration': 268,
+            'upload_date': '20141122',
+        }
     }]
 
     def _real_extract(self, url):

From 46be82b811d91be0b0876cf141e6a94e65b8fd7f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 7 May 2015 21:58:03 +0300
Subject: [PATCH 0018/2145] [vessel] Use `main_video_asset` when searching for
 video_asset (Fixes #5623)

---
 youtube_dl/extractor/vessel.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 6215f0642..3c8d2a943 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -38,9 +38,13 @@ class VesselIE(InfoExtractor):
         return req
 
     @staticmethod
-    def find_assets(data, asset_type):
+    def find_assets(data, asset_type, asset_id=None):
         for asset in data.get('assets', []):
-            if asset.get('type') == asset_type:
+            if not asset.get('type') == asset_type:
+                continue
+            elif asset_id is not None and not asset.get('id') == asset_id:
+                continue
+            else:
                 yield asset
 
     def _check_access_rights(self, data):
@@ -82,11 +86,13 @@ class VesselIE(InfoExtractor):
         req = VesselIE.make_json_request(
             self._API_URL_TEMPLATE % asset_id, {'client': 'web'})
         data = self._download_json(req, video_id)
+        video_asset_id = data.get('main_video_asset')
 
         self._check_access_rights(data)
 
         try:
-            video_asset = next(VesselIE.find_assets(data, 'video'))
+            video_asset = next(
+                VesselIE.find_assets(data, 'video', asset_id=video_asset_id))
         except StopIteration:
             raise ExtractorError('No video assets found')
 

From 156fc83a55b14258bb4a2fa1ec3b02d4db679603 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 03:02:34 +0800
Subject: [PATCH 0019/2145] [downloader/rtmp] Fix a typo

---
 youtube_dl/downloader/rtmp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6865b5e2f..7d19bb808 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -131,7 +131,7 @@ class RtmpFD(FileDownloader):
         if play_path is not None:
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
-            basic_args += ['--tcUrl', url]
+            basic_args += ['--tcUrl', tc_url]
         if test:
             basic_args += ['--stop', '1']
         if flash_version is not None:

From 7ef00afe9da87c7d0fdbea93af39b47d5447f1a0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 03:09:19 +0800
Subject: [PATCH 0020/2145] [nhl] Support RTMP videos (fixes #4481)

---
 youtube_dl/extractor/nhl.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index b572370c2..279b18386 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -50,7 +50,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
             video_url = initial_video_url
 
         join = compat_urlparse.urljoin
-        return {
+        ret = {
             'id': video_id,
             'title': info['name'],
             'url': video_url,
@@ -59,6 +59,15 @@ class NHLBaseInfoExtractor(InfoExtractor):
             'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
             'upload_date': unified_strdate(info['releaseDate'].split('.')[0]),
         }
+        if video_url.startswith('rtmp:'):
+            mobj = re.match(r'(?P<tc_url>rtmp://[^/]+/(?P<app>[a-z0-9/]+))/(?P<play_path>mp4:.*)', video_url)
+            ret.update({
+                'tc_url': mobj.group('tc_url'),
+                'play_path': mobj.group('play_path'),
+                'app': mobj.group('app'),
+                'no_resume': True,
+            })
+        return ret
 
 
 class NHLIE(NHLBaseInfoExtractor):
@@ -115,6 +124,18 @@ class NHLIE(NHLBaseInfoExtractor):
             'duration': 268,
             'upload_date': '20141122',
         }
+    }, {
+        'url': 'http://video.oilers.nhl.com/videocenter/console?id=691469&catid=4',
+        'info_dict': {
+            'id': '691469',
+            'ext': 'mp4',
+            'title': 'RAW | Craig MacTavish Full Press Conference',
+            'description': 'Oilers GM Craig MacTavish addresses the media at Rexall Place on Friday.',
+            'upload_date': '20141205',
+        },
+        'params': {
+            'skip_download': True,  # Requires rtmpdump
+        }
     }]
 
     def _real_extract(self, url):

From a745475808e125a590afb14df48c565309d3f75c Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 02:50:46 +0200
Subject: [PATCH 0021/2145] Ir90Tv Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ir90tv.py   | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/ir90tv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dfa781f8..ee05a6958 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -229,6 +229,7 @@ from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
+from .ir90tv import Ir90TvIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
new file mode 100644
index 000000000..5aa9d6ff4
--- /dev/null
+++ b/youtube_dl/extractor/ir90tv.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Ir90TvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
+    _TEST = {
+        'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'md5': '411dbd94891381960cb9e13daa47a869',
+        'info_dict': {
+            'id': '95719',
+            'ext': 'mp4',
+            'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # TODO more code goes here, for example ...
+        title = self._html_search_regex(
+            r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
+
+        video_url = self._search_regex(
+            r'<source[^>]+src="([^"]+)"', webpage, 'video url')
+
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
+        print thumbnail
+
+
+        return {
+            'url': video_url,
+            'id': video_id,
+            'title': title,
+            'video_url' : video_url,
+            'thumbnail' : thumbnail,
+        }
\ No newline at end of file

From 54b31d149e7be08eb7be9981a9eec398d11f17ef Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 02:55:01 +0200
Subject: [PATCH 0022/2145] Ir90Tv Add new extractor

---
 youtube_dl/extractor/ir90tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 5aa9d6ff4..3a3cb4887 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -38,4 +38,4 @@ class Ir90TvIE(InfoExtractor):
             'title': title,
             'video_url' : video_url,
             'thumbnail' : thumbnail,
-        }
\ No newline at end of file
+        }

From a650110ba762b2658c64392317c1afd2a284dd3d Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 04:32:08 +0200
Subject: [PATCH 0023/2145] remove print

---
 youtube_dl/extractor/ir90tv.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 3a3cb4887..b79529b1b 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -21,7 +21,6 @@ class Ir90TvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # TODO more code goes here, for example ...
         title = self._html_search_regex(
             r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
 
@@ -29,8 +28,6 @@ class Ir90TvIE(InfoExtractor):
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
         thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
-        print thumbnail
-
 
         return {
             'url': video_url,

From 541168039d8f3e7680a15cc366fcc94335308d81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 8 May 2015 11:01:24 +0200
Subject: [PATCH 0024/2145] [utils] get_exe_version: encode executable name
 (fixes #5647)

It failed in python 2.x when $PATH contains a directory with non-ascii characters.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1013f7c18..de09b53b2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1380,7 +1380,7 @@ def get_exe_version(exe, args=['--version'],
     or False if the executable is not present """
     try:
         out, _ = subprocess.Popen(
-            [exe] + args,
+            [encodeArgument(exe)] + args,
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
     except OSError:
         return False

From 249962ffa2b155795ebfe0a267eb025cd5e30c56 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 16:56:15 +0100
Subject: [PATCH 0025/2145] [bet] Use unique part of xml url as the video id
 and fix tests (closes #5642)

The guid changes often.
---
 youtube_dl/extractor/bet.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index d2abd4d77..26b934543 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,11 +16,11 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '740ab250-bb94-4a8a-8787-fe0de7c74471',
+                'id': 'news/national/2014/a-conversation-with-president-obama',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
-                'title': 'BET News Presents: A Conversation With President Obama',
-                'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
+                'title': 'A Conversation With President Obama',
+                'description': 'md5:699d0652a350cf3e491cd15cc745b5da',
                 'duration': 1534,
                 'timestamp': 1418075340,
                 'upload_date': '20141208',
@@ -35,7 +35,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d',
+                'id': 'news/national/2014/justice-for-ferguson-a-community-reacts',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
@@ -61,6 +61,9 @@ class BetIE(InfoExtractor):
             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
             webpage, 'media URL'))
 
+        video_id = self._search_regex(
+            r'/video/(.*)/_jcr_content/', media_url, 'video id')
+
         mrss = self._download_xml(media_url, display_id)
 
         item = mrss.find('./channel/item')
@@ -75,8 +78,6 @@ class BetIE(InfoExtractor):
         description = xpath_text(
             item, './description', 'description', fatal=False)
 
-        video_id = xpath_text(item, './guid', 'video id', fatal=False)
-
         timestamp = parse_iso8601(xpath_text(
             item, xpath_with_ns('./dc:date', NS_MAP),
             'upload date', fatal=False))

From 43837189c18af635cfb1cd8fe503265b4b218c32 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 8 May 2015 10:40:25 +0100
Subject: [PATCH 0026/2145] Fix URL template extraction for netzkino. Fixes
 #5614

---
 youtube_dl/extractor/netzkino.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
index bc17e20aa..0d165a82a 100644
--- a/youtube_dl/extractor/netzkino.py
+++ b/youtube_dl/extractor/netzkino.py
@@ -49,7 +49,7 @@ class NetzkinoIE(InfoExtractor):
             'http://www.netzkino.de/beta/dist/production.min.js', video_id,
             note='Downloading player code')
         avo_js = self._search_regex(
-            r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})',
+            r'var urlTemplate=(\{.*?"\})',
             production_js, 'URL templates')
         templates = self._parse_json(
             avo_js, video_id, transform_source=js_to_json)

From d1feb308116f57ceae3888db5e1b93394300f564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 20:07:53 +0600
Subject: [PATCH 0027/2145] [mlb] Fallback to extracting video id from webpage
 for all URLs that does not contain it explicitly (Closes #5630)

---
 youtube_dl/extractor/mlb.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index ee9ff73bf..109eecefd 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,21 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/(?:embed|m-internal-embed)\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/
+                        (?:
+                            (?:
+                                (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
+                                (?:
+                                    shared/video/embed/(?:embed|m-internal-embed)\.html|
+                                    [^/]+/video/play\.jsp
+                                )\?.*?\bcontent_id=
+                            )
+                            (?P<id>n?\d+)|
+                            (?P<path>.+?)
+                        )
+                    '''
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -95,6 +109,12 @@ class MLBIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        if not video_id:
+            video_path = mobj.group('path')
+            webpage = self._download_webpage(url, video_path)
+            video_id = self._search_regex(
+                r'data-videoid="(\d+)"', webpage, 'video id')
+
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'
             % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id)

From 4e6e9d21bd62c4e2ab2576347e066891092a5783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 21:48:47 +0600
Subject: [PATCH 0028/2145] [mlb] Improve _VALID_URL

---
 youtube_dl/extractor/mlb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 109eecefd..4e054fb53 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -22,7 +22,7 @@ class MLBIE(InfoExtractor):
                                 )\?.*?\bcontent_id=
                             )
                             (?P<id>n?\d+)|
-                            (?P<path>.+?)
+                            (?:[^/]+/)*(?P<path>[^/]+)
                         )
                     '''
     _TESTS = [

From 34e7dc81a94d39d48c5b4aac8cddcca46edba94d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:03:03 +0600
Subject: [PATCH 0029/2145] [vgtv] Add support for generic bt.no URLs (#5620)

---
 youtube_dl/extractor/vgtv.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 69dc9a759..b0f0b3bc2 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -8,7 +8,8 @@ from ..utils import float_or_none
 
 
 class VGTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/[^/]+/(?P<id>[0-9]+)'
+    IE_DESC = 'VGTV and BTTV'
+    _VALID_URL = r'http://(?:www\.)?(?P<host>vgtv|bt)\.no/(?:(?:tv/)?#!/(?:video|live)/(?P<id>[0-9]+)|(?:[^/]+/)*(?P<path>[^/]+))'
     _TESTS = [
         {
             # streamType: vod
@@ -64,12 +65,25 @@ class VGTVIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+
+        HOST_WEBSITES = {
+            'vgtv': 'vgtv',
+            'bt': 'bttv',
+        }
+
         data = self._download_json(
-            'http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website' % video_id,
+            'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
+            % (host, video_id, HOST_WEBSITES[host]),
             video_id, 'Downloading media JSON')
 
         streams = data['streamUrls']

From 0ceab8474924c4e7a6e28497c8da40cc5002c8d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:18:43 +0600
Subject: [PATCH 0030/2145] [vgtv] Add support for bt.no articles (#5620)

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/vgtv.py     | 39 +++++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dfa781f8..587a45940 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -587,7 +587,10 @@ from .veoh import VeohIE
 from .vessel import VesselIE
 from .vesti import VestiIE
 from .vevo import VevoIE
-from .vgtv import VGTVIE
+from .vgtv import (
+    BTArticleIE,
+    VGTVIE,
+)
 from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index b0f0b3bc2..ad07e54c9 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -9,7 +9,18 @@ from ..utils import float_or_none
 
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV and BTTV'
-    _VALID_URL = r'http://(?:www\.)?(?P<host>vgtv|bt)\.no/(?:(?:tv/)?#!/(?:video|live)/(?P<id>[0-9]+)|(?:[^/]+/)*(?P<path>[^/]+))'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        vgtv:|
+                        http://(?:www\.)?
+                    )
+                    (?P<host>vgtv|bt)
+                    (?:
+                        :|
+                        \.no/(?:tv/)?#!/(?:video|live)/
+                    )
+                    (?P<id>[0-9]+)
+                    '''
     _TESTS = [
         {
             # streamType: vod
@@ -129,3 +140,29 @@ class VGTVIE(InfoExtractor):
             'view_count': data['displays'],
             'formats': formats,
         }
+
+
+class BTArticleIE(InfoExtractor):
+    IE_DESC = 'Bergens Tidende'
+    _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _TEST = {
+        'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
+        'md5': 'd055e8ee918ef2844745fcfd1a4175fb',
+        'info_dict': {
+            'id': '23199',
+            'ext': 'mp4',
+            'title': 'Alrekstad internat',
+            'description': 'md5:dc81a9056c874fedb62fc48a300dac58',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 191,
+            'timestamp': 1289991323,
+            'upload_date': '20101117',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, self._match_id(url))
+        video_id = self._search_regex(
+            r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
+        return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')

From 2c0c9dc46cda490137b6788d6d66f31ca092f58f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:50:01 +0600
Subject: [PATCH 0031/2145] [xstream] Move xstream to separate extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/xstream.py  | 115 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/xstream.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 587a45940..5cc35c8eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -653,6 +653,7 @@ from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
+from .xstream import XstreamIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
new file mode 100644
index 000000000..71584c291
--- /dev/null
+++ b/youtube_dl/extractor/xstream.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    find_xpath_attr,
+)
+
+
+class XstreamIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        xstream:|
+                        https?://frontend\.xstream\.(?:dk|net)/
+                    )
+                    (?P<partner_id>[^/]+)
+                    (?:
+                        :|
+                        /feed/video/\?.*?\bid=
+                    )
+                    (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'http://frontend.xstream.dk/btno/feed/video/?platform=web&id=86588',
+        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
+        'info_dict': {
+            'id': '86588',
+            'ext': 'mov',
+            'title': 'Otto Wollertsen',
+            'description': 'Vestlendingen Otto Fredrik Wollertsen',
+            'timestamp': 1430473209,
+            'upload_date': '20150501',
+        },
+    }, {
+        'url': 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=21039',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+
+        data = self._download_xml(
+            'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s'
+            % (partner_id, video_id),
+            video_id)
+
+        NS_MAP = {
+            'atom': 'http://www.w3.org/2005/Atom',
+            'xt': 'http://xstream.dk/',
+            'media': 'http://search.yahoo.com/mrss/',
+        }
+
+        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
+
+        title = xpath_text(
+            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
+        description = xpath_text(
+            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
+        timestamp = parse_iso8601(xpath_text(
+            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
+
+        formats = []
+        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
+        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
+            media_url = media_content.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media_content.get('bitrate'))
+            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
+            if mobj:
+                formats.append({
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:%s' % mobj.group('playpath'),
+                    'app': mobj.group('app'),
+                    'ext': 'flv',
+                    'tbr': tbr,
+                    'format_id': 'rtmp-%d' % tbr,
+                })
+            else:
+                formats.append({
+                    'url': media_url,
+                    'tbr': tbr,
+                })
+        self._sort_formats(formats)
+
+        link = find_xpath_attr(
+            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
+        if link is not None:
+            formats.append({
+                'url': link.get('href'),
+                'format_id': link.get('rel'),
+            })
+
+        thumbnails = [{
+            'url': splash.get('url'),
+            'width': int_or_none(splash.get('width')),
+            'height': int_or_none(splash.get('height')),
+        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From cbe443362f91ab111e2a01fe8246e17a98668f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:52:20 +0600
Subject: [PATCH 0032/2145] [aftenposten] Implement in terms of xtream
 extractor

---
 youtube_dl/extractor/aftenposten.py | 77 +----------------------------
 1 file changed, 1 insertion(+), 76 deletions(-)

diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
index e15c015fb..0c00acfb5 100644
--- a/youtube_dl/extractor/aftenposten.py
+++ b/youtube_dl/extractor/aftenposten.py
@@ -1,21 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_iso8601,
-    xpath_with_ns,
-    xpath_text,
-    find_xpath_attr,
-)
 
 
 class AftenpostenIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
-
     _TEST = {
         'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
         'md5': 'fd828cd29774a729bf4d4425fe192972',
@@ -30,69 +20,4 @@ class AftenpostenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        data = self._download_xml(
-            'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)
-
-        NS_MAP = {
-            'atom': 'http://www.w3.org/2005/Atom',
-            'xt': 'http://xstream.dk/',
-            'media': 'http://search.yahoo.com/mrss/',
-        }
-
-        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
-
-        title = xpath_text(
-            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
-        description = xpath_text(
-            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
-        timestamp = parse_iso8601(xpath_text(
-            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
-
-        formats = []
-        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
-        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
-            media_url = media_content.get('url')
-            if not media_url:
-                continue
-            tbr = int_or_none(media_content.get('bitrate'))
-            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
-            if mobj:
-                formats.append({
-                    'url': mobj.group('url'),
-                    'play_path': 'mp4:%s' % mobj.group('playpath'),
-                    'app': mobj.group('app'),
-                    'ext': 'flv',
-                    'tbr': tbr,
-                    'format_id': 'rtmp-%d' % tbr,
-                })
-            else:
-                formats.append({
-                    'url': media_url,
-                    'tbr': tbr,
-                })
-        self._sort_formats(formats)
-
-        link = find_xpath_attr(
-            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
-        if link is not None:
-            formats.append({
-                'url': link.get('href'),
-                'format_id': link.get('rel'),
-            })
-
-        thumbnails = [{
-            'url': splash.get('url'),
-            'width': int_or_none(splash.get('width')),
-            'height': int_or_none(splash.get('height')),
-        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'formats': formats,
-            'thumbnails': thumbnails,
-        }
+        return self.url_result('xstream:ap:%s' % self._match_id(url), 'Xstream')

From fe373287ebdda002ed84dca1d8b9d6f8a5686138 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:59:50 +0600
Subject: [PATCH 0033/2145] [vgtv] Add support for bt vestlendingen (Closes
 #5620)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vgtv.py     | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5cc35c8eb..96cf28efe 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -589,6 +589,7 @@ from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import (
     BTArticleIE,
+    BTVestlendingenIE,
     VGTVIE,
 )
 from .vh1 import VH1IE
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index ad07e54c9..db7a4bdb1 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -143,7 +143,8 @@ class VGTVIE(InfoExtractor):
 
 
 class BTArticleIE(InfoExtractor):
-    IE_DESC = 'Bergens Tidende'
+    IE_NAME = 'bt:article'
+    IE_DESC = 'Bergens Tidende Articles'
     _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
@@ -166,3 +167,26 @@ class BTArticleIE(InfoExtractor):
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
         return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')
+
+
+class BTVestlendingenIE(InfoExtractor):
+    IE_NAME = 'bt:vestlendingen'
+    IE_DESC = 'Bergens Tidende - Vestlendingen'
+    _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
+        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
+        'info_dict': {
+            'id': '86588',
+            'ext': 'mov',
+            'title': 'Otto Wollertsen',
+            'description': 'Vestlendingen Otto Fredrik Wollertsen',
+            'timestamp': 1430473209,
+            'upload_date': '20150501',
+        },
+    }
+
+    def _real_extract(self, url):
+        return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
+
+

From 4384cf9e7d59492141ebd45f77830238097c695c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:04:27 +0600
Subject: [PATCH 0034/2145] [extractor/__init__] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 96cf28efe..7f0070784 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -655,9 +655,9 @@ from .xhamster import XHamsterIE
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
-from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
+from .xvideos import XVideosIE
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,

From bb03fdae0d4da9c591c2967044c5e30bf797c22a Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:19:57 +0600
Subject: [PATCH 0035/2145] [README.md] Clarify format selection when streaming
 to stdout

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9aeb114f3..b6e9429df 100644
--- a/README.md
+++ b/README.md
@@ -269,7 +269,7 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded an muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 

From 50b901306406d5c37f31880860e2a4dbb5e0a165 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:21:23 +0600
Subject: [PATCH 0036/2145] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b6e9429df..3d9436456 100644
--- a/README.md
+++ b/README.md
@@ -269,7 +269,7 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded an muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 

From 79998cd5afb2e16fe14cebdbec81b21c45c24c32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:12:42 +0600
Subject: [PATCH 0037/2145] [svtplay] Generalize svt extractors and add svt.se
 extractor

---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/svtplay.py  | 98 ++++++++++++++++++++------------
 2 files changed, 67 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f0070784..79236c6c1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -505,7 +505,10 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
-from .svtplay import SVTPlayIE
+from .svtplay import (
+    SVTIE,
+    SVTPlayIE,
+)
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index 433dfd1cb..732f02048 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -9,41 +9,9 @@ from ..utils import (
 )
 
 
-class SVTPlayIE(InfoExtractor):
-    IE_DESC = 'SVT Play and Öppet arkiv'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'ade3def0643fa1c40587a422f98edfd9',
-        'info_dict': {
-            'id': '2609989',
-            'ext': 'flv',
-            'title': 'SM veckan vinter, Örebro - Rally, final',
-            'duration': 4500,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-    }, {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
-
-        info = self._download_json(
-            'http://www.%s.se/video/%s?output=json' % (host, video_id), video_id)
+class SVTBaseIE(InfoExtractor):
+    def _extract_video(self, url, video_id):
+        info = self._download_json(url, video_id)
 
         title = info['context']['title']
         thumbnail = info['context'].get('thumbnailImage')
@@ -80,3 +48,63 @@ class SVTPlayIE(InfoExtractor):
             'duration': duration,
             'age_limit': age_limit,
         }
+
+
+class SVTIE(SVTBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
+        'md5': '9648197555fc1b49e3dc22db4af51d46',
+        'info_dict': {
+            'id': '2900353',
+            'ext': 'flv',
+            'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+            'duration': 27,
+            'age_limit': 0,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        widget_id = mobj.group('widget_id')
+        article_id = mobj.group('id')
+        return self._extract_video(
+            'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
+            article_id)
+
+
+class SVTPlayIE(SVTBaseIE):
+    IE_DESC = 'SVT Play and Öppet arkiv'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
+        'md5': 'ade3def0643fa1c40587a422f98edfd9',
+        'info_dict': {
+            'id': '2609989',
+            'ext': 'flv',
+            'title': 'SM veckan vinter, Örebro - Rally, final',
+            'duration': 4500,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'flv',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+        'skip': 'Only works from Sweden',
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+        return self._extract_video(
+            'http://www.%s.se/video/%s?output=json' % (host, video_id),
+            video_id)

From 322915014f0378e2675a2a17cd67fe89a6e6a7d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:13:40 +0600
Subject: [PATCH 0038/2145] [svtplay] Rename to svt

---
 youtube_dl/extractor/__init__.py            | 2 +-
 youtube_dl/extractor/{svtplay.py => svt.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{svtplay.py => svt.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79236c6c1..0a18dba5c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -505,7 +505,7 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
-from .svtplay import (
+from .svt import (
     SVTIE,
     SVTPlayIE,
 )
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svt.py
similarity index 100%
rename from youtube_dl/extractor/svtplay.py
rename to youtube_dl/extractor/svt.py

From bab19a8e91153705d6600fe1d1a0b0aa0bf93bb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:23:35 +0600
Subject: [PATCH 0039/2145] [extractor/generic] Add support for svt embeds
 (Closes #5622)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 youtube_dl/extractor/svt.py     | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cd7c47d6d..046bcb0f0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,6 +37,7 @@ from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
+from .svt import SVTIE
 
 
 class GenericIE(InfoExtractor):
@@ -1091,6 +1092,11 @@ class GenericIE(InfoExtractor):
         if bliptv_url:
             return self.url_result(bliptv_url, 'BlipTV')
 
+        # Look for SVT player
+        svt_url = SVTIE._extract_url(webpage)
+        if svt_url:
+            return self.url_result(svt_url, 'SVT')
+
         # Look for embedded condenast player
         matches = re.findall(
             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 732f02048..fc20f664b 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -64,6 +64,13 @@ class SVTIE(SVTBaseIE):
         },
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'(?:<iframe src|href)="(?P<url>%s[^"]*)"' % SVTIE._VALID_URL, webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         widget_id = mobj.group('widget_id')

From dc455a5f88d6bb9bd5a44352e2b67ef7dd5fd683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:27:37 +0600
Subject: [PATCH 0040/2145] [extractor/generic] Add test for svt embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 046bcb0f0..7c38bce7c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -659,6 +659,17 @@ class GenericIE(InfoExtractor):
                 'title': 'Facebook Creates "On This Day" | Crunch Report',
             },
         },
+        # SVT embed
+        {
+            'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
+            'info_dict': {
+                'id': '2900353',
+                'ext': 'flv',
+                'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+                'duration': 27,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',

From de765f6c3188802bb2dea704a645f539fa61c8aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 02:15:51 +0600
Subject: [PATCH 0041/2145] [foxsports] Support some more URLs (#5611)

---
 youtube_dl/extractor/foxsports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index 363866b64..df7665176 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -5,7 +5,7 @@ from ..utils import smuggle_url
 
 
 class FoxSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/video\?vid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.foxsports.com/video?vid=432609859715',

From 3dbec410a0e195036025aa3a3792932783f371d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 13:19:54 +0800
Subject: [PATCH 0042/2145] [sohu] Enhance error handling

---
 youtube_dl/extractor/sohu.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index f8a4840f7..13b9e9133 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import sanitize_url_path_consecutive_slashes
+from ..utils import (
+    sanitize_url_path_consecutive_slashes,
+    ExtractorError,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -117,6 +120,15 @@ class SohuIE(InfoExtractor):
             r'var vid ?= ?["\'](\d+)["\']',
             webpage, 'video path')
         vid_data = _fetch_data(vid, mytv)
+        if vid_data['play'] != 1:
+            if vid_data.get('status') == 12:
+                raise ExtractorError(
+                    'Sohu said: There\'s something wrong in the video.',
+                    expected=True)
+            else:
+                raise ExtractorError(
+                    'Sohu said: The video is only licensed to users in Mainland China.',
+                    expected=True)
 
         formats_json = {}
         for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):

From 32060c6d6b618fa858b2ce43db34d02fd43bc542 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 13:54:28 +0800
Subject: [PATCH 0043/2145] [sohu] Update extractor

The original extraction logic always fails for all test videos
---
 youtube_dl/extractor/sohu.py | 44 ++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 13b9e9133..eab4adfca 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,10 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import (
-    sanitize_url_path_consecutive_slashes,
-    ExtractorError,
-)
+from ..utils import ExtractorError
 
 
 class SohuIE(InfoExtractor):
@@ -31,7 +28,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': '699060e75cf58858dd47fb9c03c42cfb',
+        'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -39,7 +36,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '9bf34be48f2f4dadcb226c74127e203c',
+        'md5': '49308ff6dafde5ece51137d04aec311e',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -53,7 +50,7 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
+            'md5': '492923eac023ba2f13ff69617c32754a',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -61,7 +58,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
+            'md5': 'de604848c0e8e9c4a4dde7e1347c0637',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -69,7 +66,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '8407e634175fdac706766481b9443450',
+            'md5': '93584716ee0657c0b205b8aa3d27aa13',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',
@@ -144,24 +141,21 @@ class SohuIE(InfoExtractor):
         for i in range(part_count):
             formats = []
             for format_id, format_data in formats_json.items():
-                allot = format_data['allot']
-                prot = format_data['prot']
-
                 data = format_data['data']
-                clips_url = data['clipsURL']
-                su = data['su']
 
-                part_str = self._download_webpage(
-                    'http://%s/?prot=%s&file=%s&new=%s' %
-                    (allot, prot, clips_url[i], su[i]),
-                    video_id,
-                    'Downloading %s video URL part %d of %d'
-                    % (format_id, i + 1, part_count))
-
-                part_info = part_str.split('|')
-
-                video_url = sanitize_url_path_consecutive_slashes(
-                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
+                # URLs starts with http://newflv.sohu.ccgslb.net/ is not usable
+                # so retry until got a working URL
+                video_url = 'newflv.sohu.ccgslb.net'
+                retries = 0
+                while 'newflv.sohu.ccgslb.net' in video_url and retries < 5:
+                    download_note = 'Download information from CDN gateway for format ' + format_id
+                    if retries > 0:
+                        download_note += ' (retry #%d)' % retries
+                    retries += 1
+                    cdn_info = self._download_json(
+                        'http://data.vod.itc.cn/cdnList?new=' + data['su'][i],
+                        video_id, download_note)
+                    video_url = cdn_info['url']
 
                 formats.append({
                     'url': video_url,

From 6d14d08e062ff3d6e0fd17f04cb341099097902c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 17:36:07 +0800
Subject: [PATCH 0044/2145] [yam] Fix title and uploader id

---
 youtube_dl/extractor/yam.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 19f8762ae..9d851bae3 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     month_by_abbreviation,
     ExtractorError,
+    get_element_by_attribute,
 )
 
 
@@ -23,6 +24,7 @@ class YamIE(InfoExtractor):
             'id': '2283921',
             'ext': 'mp3',
             'title': '發現 - 趙薇 京華煙雲主題曲',
+            'description': '發現 - 趙薇 京華煙雲主題曲',
             'uploader_id': 'princekt',
             'upload_date': '20080807',
             'duration': 313.0,
@@ -55,6 +57,17 @@ class YamIE(InfoExtractor):
             'ext': 'mp4',
         },
         'skip': 'invalid YouTube URL',
+    }, {
+        'url': 'http://mymedia.yam.com/m/2373534',
+        'md5': '7ff74b91b7a817269d83796f8c5890b1',
+        'info_dict': {
+            'id': '2373534',
+            'ext': 'mp3',
+            'title': '林俊傑&蔡卓妍-小酒窩',
+            'description': 'md5:904003395a0fcce6cfb25028ff468420',
+            'upload_date': '20080928',
+            'uploader_id': 'onliner2',
+        }
     }]
 
     def _real_extract(self, url):
@@ -75,15 +88,19 @@ class YamIE(InfoExtractor):
         if youtube_url:
             return self.url_result(youtube_url, 'Youtube')
 
+        title = self._html_search_regex(
+            r'<h1[^>]+class="heading"[^>]*>\s*(.+)\s*</h1>', page, 'title')
+
         api_page = self._download_webpage(
             'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
             note='Downloading API page')
         api_result_obj = compat_urlparse.parse_qs(api_page)
 
+        info_table = get_element_by_attribute('class', 'info', page)
         uploader_id = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"',
-            page, 'uploader id', fatal=False)
-        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z0-9]+)"',
+            info_table, 'uploader id', fatal=False)
+        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})\s+' +
                          r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
         if mobj:
             upload_date = '%s%02d%02d' % (
@@ -97,7 +114,8 @@ class YamIE(InfoExtractor):
         return {
             'id': video_id,
             'url': api_result_obj['mp3file'][0],
-            'title': self._html_search_meta('description', page),
+            'title': title,
+            'description': self._html_search_meta('description', page),
             'duration': duration,
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From d39e0f05db226ef5691f5730d40da796aec6bac6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 17:37:39 +0800
Subject: [PATCH 0045/2145] [utils] Remove
 sanitize_url_path_consecutive_slashes()

This function is used only in SohuIE, which is updated to use a new
extraction logic.
---
 test/test_utils.py  | 21 ---------------------
 youtube_dl/utils.py |  7 -------
 2 files changed, 28 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 032d3656a..86b110a7d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -40,7 +40,6 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
-    sanitize_url_path_consecutive_slashes,
     prepend_extension,
     replace_extension,
     shell_quote,
@@ -176,26 +175,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
-    def test_sanitize_url_path_consecutive_slashes(self):
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/foo//bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname//foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname//'),
-            'http://hostname/')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/'),
-            'http://hostname/')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
-            'http://hostname/abc/')
-
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
         self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index de09b53b2..d73efcf25 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -327,13 +327,6 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
-def sanitize_url_path_consecutive_slashes(url):
-    """Collapses consecutive slashes in URLs' path"""
-    parsed_url = list(compat_urlparse.urlparse(url))
-    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
-    return compat_urlparse.urlunparse(parsed_url)
-
-
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From 5c0b2c16a80c509dbcee48f48da3de0bf9912cda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 May 2015 12:34:45 +0200
Subject: [PATCH 0046/2145] [vgtv] Escape '#' in _VALID_URL and remove empty
 newlines at the end

In verbose mode, '#' is interpreted as the start of a comment.
---
 youtube_dl/extractor/vgtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index db7a4bdb1..eb2652fb1 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
                     (?P<host>vgtv|bt)
                     (?:
                         :|
-                        \.no/(?:tv/)?#!/(?:video|live)/
+                        \.no/(?:tv/)?\#!/(?:video|live)/
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -188,5 +188,3 @@ class BTVestlendingenIE(InfoExtractor):
 
     def _real_extract(self, url):
         return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
-
-

From 3b5f65a64c06859cdee0b93f319c80d5c116cedd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 May 2015 12:41:56 +0200
Subject: [PATCH 0047/2145] [mlb] Fix extraction of articles

And move test from generic, since it's directly handled by MLBIE
---
 youtube_dl/extractor/generic.py | 13 -------------
 youtube_dl/extractor/mlb.py     | 14 +++++++++++++-
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7c38bce7c..3d756e848 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -414,19 +414,6 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
-        # MLB articles
-        {
-            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
-            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
-            'info_dict': {
-                'id': '75609783',
-                'ext': 'mp4',
-                'title': 'Must C: Pillar climbs for catch',
-                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
-                'timestamp': 1429124820,
-                'upload_date': '20150415',
-            }
-        },
         # Wistia embed
         {
             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 4e054fb53..40c9ecb35 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -82,6 +82,18 @@ class MLBIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        {
+            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
+            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
+            'info_dict': {
+                'id': '75609783',
+                'ext': 'mp4',
+                'title': 'Must C: Pillar climbs for catch',
+                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
+                'timestamp': 1429124820,
+                'upload_date': '20150415',
+            }
+        },
         {
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
@@ -113,7 +125,7 @@ class MLBIE(InfoExtractor):
             video_path = mobj.group('path')
             webpage = self._download_webpage(url, video_path)
             video_id = self._search_regex(
-                r'data-videoid="(\d+)"', webpage, 'video id')
+                r'data-video-?id="(\d+)"', webpage, 'video id')
 
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'

From d592b42f5c37c4b0d9f1587dd76f225b0287959f Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Sat, 9 May 2015 15:26:00 +0100
Subject: [PATCH 0048/2145] Updated two tests for BRIE

---
 youtube_dl/extractor/br.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 45ba51732..04a3ecd17 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -16,27 +16,27 @@ class BRIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.br.de/mediathek/video/sendungen/heimatsound/heimatsound-festival-2014-trailer-100.html',
-            'md5': '93556dd2bcb2948d9259f8670c516d59',
+            'url': 'http://www.br.de/mediathek/video/sendungen/abendschau/betriebliche-altersvorsorge-104.html',
+            'md5': '83a0477cf0b8451027eb566d88b51106',
             'info_dict': {
-                'id': '25e279aa-1ffd-40fd-9955-5325bd48a53a',
+                'id': '48f656ef-287e-486f-be86-459122db22cc',
                 'ext': 'mp4',
-                'title': 'Wenn das Traditions-Theater wackelt',
-                'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt',
-                'duration': 34,
-                'uploader': 'BR',
-                'upload_date': '20140802',
+                'title': 'Die böse Überraschung',
+                'description': 'Betriebliche Altersvorsorge: Die böse Überraschung',
+                'duration': 180,
+                'uploader': 'Reinhard Weber',
+                'upload_date': '20150422',
             }
         },
         {
-            'url': 'http://www.br.de/nachrichten/schaeuble-haushaltsentwurf-bundestag-100.html',
-            'md5': '3db0df1a9a9cd9fa0c70e6ea8aa8e820',
+            'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
+            'md5': 'a44396d73ab6a68a69a568fae10705bb',
             'info_dict': {
-                'id': 'c6aae3de-2cf9-43f2-957f-f17fef9afaab',
-                'ext': 'aac',
-                'title': '"Keine neuen Schulden im nächsten Jahr"',
-                'description': 'Haushaltsentwurf: "Keine neuen Schulden im nächsten Jahr"',
-                'duration': 64,
+                'id': 'a4b83e34-123d-4b81-9f4e-c0d3121a4e05',
+                'ext': 'mp4',
+                'title': 'Manfred Schreiber ist tot',
+                'description': 'Abendschau kompakt: Manfred Schreiber ist tot',
+                'duration': 26,
             }
         },
         {

From 0892090a56e04726175c247d13ecce7f6c9cb839 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Sat, 9 May 2015 16:02:07 +0100
Subject: [PATCH 0049/2145] Added audio test for BRIE

---
 youtube_dl/extractor/br.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 04a3ecd17..66e394e10 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -39,6 +39,17 @@ class BRIE(InfoExtractor):
                 'duration': 26,
             }
         },
+        {
+            'url': 'http://www.br.de/radio/br-klassik/sendungen/allegro/premiere-urauffuehrung-the-land-2015-dance-festival-muenchen-100.html',
+            'md5': '8b5b27c0b090f3b35eac4ab3f7a73d3d',
+            'info_dict': {
+                'id': '74c603c9-26d3-48bb-b85b-079aeed66e0b',
+                'ext': 'aac',
+                'title': 'Kurzweilig und sehr bewegend',
+                'description': '"The Land" von Peeping Tom: Kurzweilig und sehr bewegend',
+                'duration': 296,
+            }
+        },
         {
             'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',
             'md5': 'dbab0aef2e047060ea7a21fc1ce1078a',

From 32fffff2ccc044c639c8723281981aa347423762 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 21:19:09 +0600
Subject: [PATCH 0050/2145] [eroprofile] Fix video URL extraction (Closes
 #5657)

---
 youtube_dl/extractor/eroprofile.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 0cbca90b0..316033cf1 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -4,7 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    unescapeHTML
+)
 
 
 class EroProfileIE(InfoExtractor):
@@ -75,8 +78,8 @@ class EroProfileIE(InfoExtractor):
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)
 
-        video_url = self._search_regex(
-            r'<source src="([^"]+)', webpage, 'video url')
+        video_url = unescapeHTML(self._search_regex(
+            r'<source src="([^"]+)', webpage, 'video url'))
         title = self._html_search_regex(
             r'Title:</th><td>([^<]+)</td>', webpage, 'title')
         thumbnail = self._search_regex(

From f2e0056579ac507b776ce2c86b5281fc28bbc275 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 21:23:09 +0600
Subject: [PATCH 0051/2145] [vgtv] Avoid duplicate format_id

---
 youtube_dl/extractor/vgtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index eb2652fb1..e6ee1e471 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -103,11 +103,14 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4'))
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls'))
 
         hds_url = streams.get('hds')
         if hds_url:
-            formats.extend(self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id))
+            formats.extend(self._extract_f4m_formats(
+                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                video_id, f4m_id='hds'))
 
         mp4_url = streams.get('mp4')
         if mp4_url:

From 480065172d4c97f00973b3f0bf24cd1b8e567627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 00:26:42 +0600
Subject: [PATCH 0052/2145] [lifenews] Add support for video URLs (Closes
 #5660)

---
 youtube_dl/extractor/lifenews.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 081016b80..92031e843 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -14,7 +14,7 @@ from ..utils import (
 class LifeNewsIE(InfoExtractor):
     IE_NAME = 'lifenews'
     IE_DESC = 'LIFE | NEWS'
-    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?news/(?P<id>\d+)'
+    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://lifenews.ru/news/126342',
@@ -55,12 +55,15 @@ class LifeNewsIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        section = mobj.group('section')
 
-        webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
+        webpage = self._download_webpage(
+            'http://lifenews.ru/%s/%s' % (section, video_id),
+            video_id, 'Downloading page')
 
         videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
         iframe_link = self._html_search_regex(
-            '<iframe[^>]+src="([^"]+)', webpage, 'iframe link', default=None)
+            '<iframe[^>]+src=["\']([^"\']+)["\']', webpage, 'iframe link', default=None)
         if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 

From 057ebeaca3da40b901b2592e2302a0d4bbab48e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 00:27:49 +0600
Subject: [PATCH 0053/2145] [lifenews] Add test for #5660

---
 youtube_dl/extractor/lifenews.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 92031e843..7d5b8621b 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -50,6 +50,9 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150505',
             'uploader': 'embed.life.ru',
         }
+    }, {
+        'url': 'http://lifenews.ru/video/13035',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 754270313a2b337eda98fa95232bd1064e294173 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 01:03:26 +0600
Subject: [PATCH 0054/2145] [life:embed] Move to separated extractor and
 extract m3u8 formats

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/lifenews.py | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a18dba5c..f117578a2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -258,7 +258,10 @@ from .letv import (
     LetvPlaylistIE
 )
 from .libsyn import LibsynIE
-from .lifenews import LifeNewsIE
+from .lifenews import (
+    LifeNewsIE,
+    LifeEmbedIE,
+)
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 7d5b8621b..7f39fa4cf 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -4,7 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
+    determine_ext,
     int_or_none,
     unified_strdate,
     ExtractorError,
@@ -119,3 +121,48 @@ class LifeNewsIE(InfoExtractor):
             return make_entry(video_id, videos[0])
         else:
             return [make_entry(video_id, media, video_number + 1) for video_number, media in enumerate(videos)]
+
+
+class LifeEmbedIE(InfoExtractor):
+    IE_NAME = 'life:embed'
+    _VALID_URL = r'http://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
+
+    _TEST = {
+        'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
+        'md5': 'b889715c9e49cb1981281d0e5458fbbe',
+        'info_dict': {
+            'id': 'e50c2dec2867350528e2574c899b8291',
+            'ext': 'mp4',
+            'title': 'e50c2dec2867350528e2574c899b8291',
+            'thumbnail': 're:http://.*\.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for video_url in re.findall(r'"file"\s*:\s*"([^"]+)', webpage):
+            video_url = compat_urlparse.urljoin(url, video_url)
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='m3u8'))
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': ext,
+                    'preference': 1,
+                })
+
+        thumbnail = self._search_regex(
+            r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)
+
+        return {
+            'id': video_id,
+            'title': video_id,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 69fe3a5f0961c0ae602da531d2b0fb3f11b9d7c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 10 May 2015 01:05:24 +0200
Subject: [PATCH 0055/2145] release 2015.05.10

---
 docs/supportedsites.md | 9 ++++++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6d2e496a8..98b625380 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -64,6 +64,8 @@
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
  - **Brightcove**
+ - **bt:article**: Bergens Tidende Articles
+ - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
  - **BYUtv**
  - **Camdemy**
@@ -240,6 +242,7 @@
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
+ - **life:embed**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
@@ -328,6 +331,7 @@
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
+ - **NYTimesArticle**
  - **ocw.mit.edu**
  - **Odnoklassniki**
  - **OktoberfestTV**
@@ -434,6 +438,7 @@
  - **southpark.cc.com**
  - **southpark.cc.com:español**
  - **southpark.de**
+ - **southpark.nl**
  - **Space**
  - **SpankBang**
  - **Spankwire**
@@ -453,6 +458,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
+ - **SVT**
  - **SVTPlay**: SVT Play and Öppet arkiv
  - **SWRMediathek**
  - **Syfy**
@@ -529,7 +535,7 @@
  - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
- - **VGTV**
+ - **VGTV**: VGTV and BTTV
  - **vh1.com**
  - **Vice**
  - **Viddler**
@@ -587,6 +593,7 @@
  - **XHamster**
  - **XMinus**
  - **XNXX**
+ - **Xstream**
  - **XTube**
  - **XTubeUser**: XTube user profile
  - **Xuite**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a5a81bcd2..83c5a1659 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.04'
+__version__ = '2015.05.10'

From 3800b908b1976f242b41d5a2d114418559ce3b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 06:14:34 +0600
Subject: [PATCH 0056/2145] [mlb] Fix #5663

---
 youtube_dl/extractor/mlb.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 40c9ecb35..e242b897f 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -12,13 +12,13 @@ from ..utils import (
 class MLBIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/
+                        (?:[\da-z_-]+\.)*mlb\.com/
                         (?:
                             (?:
                                 (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
                                 (?:
                                     shared/video/embed/(?:embed|m-internal-embed)\.html|
-                                    [^/]+/video/play\.jsp
+                                    (?:[^/]+/)+(?:play|index)\.jsp|
                                 )\?.*?\bcontent_id=
                             )
                             (?P<id>n?\d+)|
@@ -114,6 +114,10 @@ class MLBIE(InfoExtractor):
             # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
             'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
             'only_matching': True,
+        },
+        {
+            'url': 'http://washington.nationals.mlb.com/mlb/gameday/index.jsp?c_id=was&gid=2015_05_09_atlmlb_wasmlb_1&lang=en&content_id=108309983&mode=video#',
+            'only_matching': True,
         }
     ]
 
@@ -125,7 +129,7 @@ class MLBIE(InfoExtractor):
             video_path = mobj.group('path')
             webpage = self._download_webpage(url, video_path)
             video_id = self._search_regex(
-                r'data-video-?id="(\d+)"', webpage, 'video id')
+                [r'data-video-?id="(\d+)"', r'content_id=(\d+)'], webpage, 'video id')
 
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'

From c6ddbdb66c5d6ead5e198013c54ef53d641063f1 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 12:30:07 +1200
Subject: [PATCH 0057/2145] [voicerepublic] Add new extractor

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/voicerepublic.py | 55 +++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/voicerepublic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f117578a2..5cb3c304d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -634,6 +634,7 @@ from .vk import (
     VKUserVideosIE,
 )
 from .vodlocker import VodlockerIE
+from .voicerepublic import VoiceRepublicIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vube import VubeIE
diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
new file mode 100644
index 000000000..1a90693cb
--- /dev/null
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_urllib_request,
+)
+
+
+class VoiceRepublicIE(InfoExtractor):
+    _VALID_URL = r'https?://voicerepublic\.com/talks/(?P<id>[0-9a-z-]+)'
+    _TEST = {
+        'url': 'https://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
+        'md5': '0554a24d1657915aa8e8f84e15dc9353',
+        'info_dict': {
+            'id': '2296',
+            'ext': 'm4a',
+            'title': 'Watching the Watchers: Building a Sousveillance State',
+            'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
+            'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'creator': 'M. C. McGrath',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        req = compat_urllib_request.Request(url)
+        # Older versions of Firefox get redirected to an "upgrade browser" page
+        req.add_header('User-Agent', 'youtube-dl')
+        webpage = self._download_webpage(req, display_id)
+        thumbnail = self._og_search_thumbnail(webpage)
+        video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
+
+        if '<div class=\'vr-player jp-jplayer\'' in webpage:
+            formats = [{
+                'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+                'ext': ext,
+                'format_id': ext,
+                'vcodec': 'none',
+            } for ext in ['m4a', 'mp3', 'ogg']]
+            self._sort_formats(formats)
+        else:
+            # Audio is still queued for processing
+            formats = []
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'url': self._og_search_url(webpage),
+            'thumbnail': thumbnail,
+            'description': self._og_search_description(webpage),
+            'creator': self._search_regex(r'<meta content=\'([^\']*?)\' name=\'author\'>', webpage, 'author', fatal=False),
+        }

From 95eb1adda8692a61db639fb21344ad22d1847044 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 08:54:50 +0600
Subject: [PATCH 0058/2145] [life:embed] Sort formats

---
 youtube_dl/extractor/lifenews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 7f39fa4cf..42cb6e35f 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -156,6 +156,7 @@ class LifeEmbedIE(InfoExtractor):
                     'format_id': ext,
                     'preference': 1,
                 })
+        self._sort_formats(formats)
 
         thumbnail = self._search_regex(
             r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)

From f900dc3fb9e17e399b0f33925ee239696cc46010 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:01:58 +1200
Subject: [PATCH 0059/2145] [voicerepublic] Extract author using
 _html_search_meta

---
 youtube_dl/extractor/voicerepublic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 1a90693cb..7d255d6fa 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -51,5 +51,5 @@ class VoiceRepublicIE(InfoExtractor):
             'url': self._og_search_url(webpage),
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'creator': self._search_regex(r'<meta content=\'([^\']*?)\' name=\'author\'>', webpage, 'author', fatal=False),
+            'creator': self._html_search_meta('author', webpage),
         }

From 03f760b1c0478c1f65cf6e978d7592be46873313 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:40:09 +1200
Subject: [PATCH 0060/2145] [voicerepublic] Remove creator field

---
 youtube_dl/extractor/voicerepublic.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 7d255d6fa..960974e16 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -19,7 +19,6 @@ class VoiceRepublicIE(InfoExtractor):
             'title': 'Watching the Watchers: Building a Sousveillance State',
             'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
             'description': 'md5:715ba964958afa2398df615809cfecb1',
-            'creator': 'M. C. McGrath',
         }
     }
 
@@ -51,5 +50,4 @@ class VoiceRepublicIE(InfoExtractor):
             'url': self._og_search_url(webpage),
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'creator': self._html_search_meta('author', webpage),
         }

From f03a8a3c4ec4dc95164c12181ffc1ddcb7583ef6 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:12:29 +1200
Subject: [PATCH 0061/2145] [voicerepublic] Raise ExtractorError if audio is
 still being processed

---
 youtube_dl/extractor/voicerepublic.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 960974e16..d3e35a815 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -2,10 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-from ..compat import (
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class VoiceRepublicIE(InfoExtractor):
@@ -31,17 +29,16 @@ class VoiceRepublicIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
 
-        if '<div class=\'vr-player jp-jplayer\'' in webpage:
-            formats = [{
-                'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
-                'ext': ext,
-                'format_id': ext,
-                'vcodec': 'none',
-            } for ext in ['m4a', 'mp3', 'ogg']]
-            self._sort_formats(formats)
-        else:
-            # Audio is still queued for processing
-            formats = []
+        if '<a>Queued for processing, please stand by...</a>' in webpage:
+            raise ExtractorError('Audio is still queued for processing')
+
+        formats = [{
+            'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+            'ext': ext,
+            'format_id': ext,
+            'vcodec': 'none',
+        } for ext in ['m4a', 'mp3', 'ogg']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 28ebef0b1b1b7b97137fbd8e093c09cb51954606 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 16:03:09 +1200
Subject: [PATCH 0062/2145] [voicerepublic] Detect list of available formats
 from the web page

---
 youtube_dl/extractor/voicerepublic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index d3e35a815..d150b5b5e 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import ExtractorError
@@ -32,12 +34,15 @@ class VoiceRepublicIE(InfoExtractor):
         if '<a>Queued for processing, please stand by...</a>' in webpage:
             raise ExtractorError('Audio is still queued for processing')
 
+        ext_matches = re.finditer(r'data-\w+=\'/vrmedia/\d+-clean\.(\w+)\'', webpage)
+        exts = [match.group(1) for match in ext_matches]
+
         formats = [{
             'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
             'ext': ext,
             'format_id': ext,
             'vcodec': 'none',
-        } for ext in ['m4a', 'mp3', 'ogg']]
+        } for ext in exts]
         self._sort_formats(formats)
 
         return {

From 1dcb52188d3709711b3ea5ae1ff6bdb985e79c62 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 16:38:26 +1200
Subject: [PATCH 0063/2145] [voicerepublic] Remove hardcoded paths to media
 files

---
 youtube_dl/extractor/voicerepublic.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index d150b5b5e..a3e40b940 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -34,15 +34,12 @@ class VoiceRepublicIE(InfoExtractor):
         if '<a>Queued for processing, please stand by...</a>' in webpage:
             raise ExtractorError('Audio is still queued for processing')
 
-        ext_matches = re.finditer(r'data-\w+=\'/vrmedia/\d+-clean\.(\w+)\'', webpage)
-        exts = [match.group(1) for match in ext_matches]
-
         formats = [{
-            'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+            'url': 'https://voicerepublic.com' + path,
             'ext': ext,
             'format_id': ext,
             'vcodec': 'none',
-        } for ext in exts]
+        } for ext, path in re.findall(r"data-([^=]+)='(/[^']+\.\1)'", webpage)]
         self._sort_formats(formats)
 
         return {

From a909e6ad43f9d9661691739a810d7b8853e17175 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 10 May 2015 15:27:55 +0800
Subject: [PATCH 0064/2145] [dailymotion] Patch upload_date detection.

(closes #5665)
---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index aa595af20..db10b8d00 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -52,6 +52,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'ext': 'mp4',
                 'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
+                'upload_date': '20150306',
             }
         },
         # Vevo video
@@ -106,9 +107,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         age_limit = self._rta_search(webpage)
 
         video_upload_date = None
-        mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
+        mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
         if mobj is not None:
-            video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
+            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
 
         embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
         embed_request = self._build_request(embed_url)

From 1934f3a0eaf16ae1d1644178b7128806b8629867 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 10 May 2015 18:22:07 +0800
Subject: [PATCH 0065/2145] [ndr] Extended to support n-joy.de as well (closes
 #4527)

According to http://en.wikipedia.org/wiki/N-Joy, n-joy.de is a service
hosted by NDR, so I put them together.
---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/ndr.py      | 90 ++++++++++++++++++++------------
 2 files changed, 61 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f117578a2..66adb4de7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -324,7 +324,10 @@ from .nbc import (
     NBCSportsIE,
     NBCSportsVPlayerIE,
 )
-from .ndr import NDRIE
+from .ndr import (
+    NDRIE,
+    NJoyIE,
+)
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index f49c66690..afb9eda27 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -8,41 +8,11 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     qualities,
+    parse_duration,
 )
 
 
-class NDRIE(InfoExtractor):
-    IE_NAME = 'ndr'
-    IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
-
-    _TESTS = [
-        {
-            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
-            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
-            'note': 'Video file',
-            'info_dict': {
-                'id': '25866',
-                'ext': 'mp4',
-                'title': 'Kartoffeltage in der Lewitz',
-                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
-                'duration': 166,
-            }
-        },
-        {
-            'url': 'http://www.ndr.de/info/audio51535.html',
-            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
-            'note': 'Audio file',
-            'info_dict': {
-                'id': '51535',
-                'ext': 'mp3',
-                'title': 'La Valette entgeht der Hinrichtung',
-                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
-                'duration': 884,
-            }
-        }
-    ]
-
+class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -54,7 +24,11 @@ class NDRIE(InfoExtractor):
         if description:
             description = description.strip()
 
-        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', fatal=False))
+        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', default=None))
+        if not duration:
+            duration = parse_duration(self._html_search_regex(
+                r'(<span class="min">\d+</span>:<span class="sec">\d+</span>)',
+                page, 'duration', default=None))
 
         formats = []
 
@@ -92,3 +66,53 @@ class NDRIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+
+class NDRIE(NDRBaseIE):
+    IE_NAME = 'ndr'
+    IE_DESC = 'NDR.de - Mediathek'
+    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
+
+    _TESTS = [
+        {
+            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
+            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
+            'note': 'Video file',
+            'info_dict': {
+                'id': '25866',
+                'ext': 'mp4',
+                'title': 'Kartoffeltage in der Lewitz',
+                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
+                'duration': 166,
+            }
+        },
+        {
+            'url': 'http://www.ndr.de/info/audio51535.html',
+            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+            'note': 'Audio file',
+            'info_dict': {
+                'id': '51535',
+                'ext': 'mp3',
+                'title': 'La Valette entgeht der Hinrichtung',
+                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
+                'duration': 884,
+            }
+        }
+    ]
+
+
+class NJoyIE(NDRBaseIE):
+    IE_NAME = 'N-JOY'
+    _VALID_URL = r'https?://www\.n-joy\.de/.+?(?P<id>\d+)\.html'
+
+    _TEST = {
+        'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': '2480',
+            'ext': 'mp4',
+            'title': 'Benaissa beim NDR Comedy Contest',
+            'description': 'Von seinem sehr "behaarten" Leben lässt sich Benaissa trotz aller Schwierigkeiten nicht unterkriegen.',
+            'duration': 654,
+        }
+    }

From 63cbd19f500eb4d90c1fc7c09f04de5df43a6a04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 10 May 2015 18:30:26 +0800
Subject: [PATCH 0066/2145] [ndr] Replace the 404 test case

---
 youtube_dl/extractor/ndr.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index afb9eda27..79a13958b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -84,7 +84,19 @@ class NDRIE(NDRBaseIE):
                 'title': 'Kartoffeltage in der Lewitz',
                 'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
                 'duration': 166,
-            }
+            },
+            'skip': '404 Not found',
+        },
+        {
+            'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
+            'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
+            'info_dict': {
+                'id': '988',
+                'ext': 'mp4',
+                'title': 'Party, Pötte und Parade',
+                'description': 'Hunderttausende feiern zwischen Speicherstadt und St. Pauli den 826. Hafengeburtstag. Die NDR Sondersendung zeigt die schönsten und spektakulärsten Bilder vom Auftakt.',
+                'duration': 3498,
+            },
         },
         {
             'url': 'http://www.ndr.de/info/audio51535.html',

From a6762c4a22325b5b69770de82df8725d2eb5c3df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 18:29:15 +0600
Subject: [PATCH 0067/2145] [voicerepublic] Make more robust and extract more
 metadata

---
 youtube_dl/extractor/voicerepublic.py | 95 ++++++++++++++++++++-------
 1 file changed, 71 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index a3e40b940..1106c655b 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -1,52 +1,99 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    int_or_none,
+)
 
 
 class VoiceRepublicIE(InfoExtractor):
-    _VALID_URL = r'https?://voicerepublic\.com/talks/(?P<id>[0-9a-z-]+)'
-    _TEST = {
-        'url': 'https://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
+    _VALID_URL = r'https?://voicerepublic\.com/(?:talks|embed)/(?P<id>[0-9a-z-]+)'
+    _TESTS = [{
+        'url': 'http://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
         'md5': '0554a24d1657915aa8e8f84e15dc9353',
         'info_dict': {
             'id': '2296',
+            'display_id': 'watching-the-watchers-building-a-sousveillance-state',
             'ext': 'm4a',
             'title': 'Watching the Watchers: Building a Sousveillance State',
-            'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
             'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
+            'duration': 1800,
+            'view_count': int,
         }
-    }
+    }, {
+        'url': 'http://voicerepublic.com/embed/watching-the-watchers-building-a-sousveillance-state',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        req = compat_urllib_request.Request(url)
+
+        req = compat_urllib_request.Request(
+            compat_urlparse.urljoin(url, '/talks/%s' % display_id))
         # Older versions of Firefox get redirected to an "upgrade browser" page
         req.add_header('User-Agent', 'youtube-dl')
         webpage = self._download_webpage(req, display_id)
-        thumbnail = self._og_search_thumbnail(webpage)
-        video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
 
-        if '<a>Queued for processing, please stand by...</a>' in webpage:
-            raise ExtractorError('Audio is still queued for processing')
+        if '>Queued for processing, please stand by...<' in webpage:
+            raise ExtractorError(
+                'Audio is still queued for processing', expected=True)
 
-        formats = [{
-            'url': 'https://voicerepublic.com' + path,
-            'ext': ext,
-            'format_id': ext,
-            'vcodec': 'none',
-        } for ext, path in re.findall(r"data-([^=]+)='(/[^']+\.\1)'", webpage)]
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)return ({.+?});\s*\n', webpage,
+                'data', default=None),
+            display_id, fatal=False)
+
+        if data:
+            title = data['title']
+            description = data.get('teaser')
+            talk_id = data.get('talk_id') or display_id
+            talk = data['talk']
+            duration = int_or_none(talk.get('duration'))
+            formats = [{
+                'url': compat_urlparse.urljoin(url, talk_url),
+                'format_id': format_id,
+                'ext': determine_ext(talk_url) or format_id,
+                'vcodec': 'none',
+            } for format_id, talk_url in talk['links'].items()]
+        else:
+            title = self._og_search_title(webpage)
+            description = self._html_search_regex(
+                r"(?s)<div class='talk-teaser'[^>]*>(.+?)</div>",
+                webpage, 'description', fatal=False)
+            talk_id = self._search_regex(
+                [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
+                webpage, 'talk id', default=None) or display_id
+            duration = None
+            formats = [{
+                'url': compat_urlparse.urljoin(url, talk_url),
+                'format_id': format_id,
+                'ext': determine_ext(talk_url) or format_id,
+                'vcodec': 'none',
+            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", webpage)]
         self._sort_formats(formats)
 
+        thumbnail = self._og_search_thumbnail(webpage)
+        view_count = int_or_none(self._search_regex(
+            r"class='play-count[^']*'>\s*(\d+) plays",
+            webpage, 'play count', fatal=False))
+
         return {
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-            'formats': formats,
-            'url': self._og_search_url(webpage),
+            'id': talk_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
             'thumbnail': thumbnail,
-            'description': self._og_search_description(webpage),
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
         }

From 370b39e8ece9f475d489eda721130eec9a9f15e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 18:37:52 +0600
Subject: [PATCH 0068/2145] [voicerepublic] Fix fallback branch formats
 extraction

---
 youtube_dl/extractor/voicerepublic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 1106c655b..254383d6c 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -47,12 +47,10 @@ class VoiceRepublicIE(InfoExtractor):
             raise ExtractorError(
                 'Audio is still queued for processing', expected=True)
 
-        data = self._parse_json(
-            self._search_regex(
-                r'(?s)return ({.+?});\s*\n', webpage,
-                'data', default=None),
-            display_id, fatal=False)
-
+        config = self._search_regex(
+            r'(?s)return ({.+?});\s*\n', webpage,
+            'data', default=None)
+        data = self._parse_json(config, display_id, fatal=False) if config else None
         if data:
             title = data['title']
             description = data.get('teaser')
@@ -74,12 +72,14 @@ class VoiceRepublicIE(InfoExtractor):
                 [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
                 webpage, 'talk id', default=None) or display_id
             duration = None
+            player = self._search_regex(
+                r"class='vr-player jp-jplayer'([^>]+)>", webpage, 'player')
             formats = [{
                 'url': compat_urlparse.urljoin(url, talk_url),
                 'format_id': format_id,
                 'ext': determine_ext(talk_url) or format_id,
                 'vcodec': 'none',
-            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", webpage)]
+            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", player)]
         self._sort_formats(formats)
 
         thumbnail = self._og_search_thumbnail(webpage)

From 95c5534f8ed016a81f715f291ab09c4ea2c3679c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 May 2015 17:41:11 +0200
Subject: [PATCH 0069/2145] ExecAfterDownloadPP, YoutubeDL: remove unused
 parameters

---
 youtube_dl/YoutubeDL.py                       | 1 -
 youtube_dl/__init__.py                        | 2 --
 youtube_dl/postprocessor/execafterdownload.py | 3 +--
 3 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d8583a8eb..4cf83c510 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -260,7 +260,6 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    exec_cmd:          Arbitrary command to run after downloading
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c88489f29..9cc9f851f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -246,7 +246,6 @@ def _real_main(argv=None):
     if opts.exec_cmd:
         postprocessors.append({
             'key': 'ExecAfterDownload',
-            'verboseOutput': opts.verbose,
             'exec_cmd': opts.exec_cmd,
         })
     if opts.xattr_set_filesize:
@@ -345,7 +344,6 @@ def _real_main(argv=None):
         'default_search': opts.default_search,
         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
         'encoding': opts.encoding,
-        'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 341437575..765fd8fe4 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -8,8 +8,7 @@ from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
-    def __init__(self, downloader=None, verboseOutput=None, exec_cmd=None):
-        self.verboseOutput = verboseOutput
+    def __init__(self, downloader=None, exec_cmd=None):
         self.exec_cmd = exec_cmd
 
     def run(self, information):

From 69b46b3d956220e4b3a3d5eda55768753a67f19d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 May 2015 17:47:49 +0200
Subject: [PATCH 0070/2145] ExecAfterDownloadPP: fix __init__ method

---
 youtube_dl/postprocessor/execafterdownload.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 765fd8fe4..13794b7ba 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -8,7 +8,8 @@ from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
-    def __init__(self, downloader=None, exec_cmd=None):
+    def __init__(self, downloader, exec_cmd):
+        super(ExecAfterDownloadPP, self).__init__(downloader)
         self.exec_cmd = exec_cmd
 
     def run(self, information):

From 70484b9f8ae629ccb87e8c0569f8f4bf2dfdb0ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 00:26:39 +0600
Subject: [PATCH 0071/2145] [postprocessor/ffmpeg] Extract `check_outdated`
 method

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 214de39f9..211faf69a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -36,7 +36,9 @@ class FFmpegPostProcessor(PostProcessor):
     def check_version(self):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
+        self.check_outdated()
 
+    def check_outdated(self):
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
@@ -44,6 +46,8 @@ class FFmpegPostProcessor(PostProcessor):
                 self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
+            return True
+        return False
 
     @staticmethod
     def get_versions(downloader=None):

From 7fcb605b82796e79a5f559624808ca9404df1154 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 00:27:29 +0600
Subject: [PATCH 0072/2145] [YoutubeDL] Fallback to `-f best` when merger is
 outdated

---
 youtube_dl/YoutubeDL.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cf83c510..7c3bdb964 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,9 +1086,10 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
-                    and info_dict['extractor'] in ['youtube', 'ted']
-                    and FFmpegMergerPP(self).available):
-                req_format_list.append('bestvideo+bestaudio')
+                    and info_dict['extractor'] in ['youtube', 'ted']):
+                merger = FFmpegMergerPP(self)
+                if merger.available and not merger.check_outdated():
+                    req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
         formats_to_download = []

From 13763ce599c8fbba43e57d2d79a9b007cfbd4ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 02:00:31 +0600
Subject: [PATCH 0073/2145] [postprocessor/ffmpeg] Add `can_merge` method

---
 youtube_dl/postprocessor/ffmpeg.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 211faf69a..cc65b34e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -36,9 +36,7 @@ class FFmpegPostProcessor(PostProcessor):
     def check_version(self):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
-        self.check_outdated()
 
-    def check_outdated(self):
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
@@ -46,8 +44,6 @@ class FFmpegPostProcessor(PostProcessor):
                 self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
-            return True
-        return False
 
     @staticmethod
     def get_versions(downloader=None):
@@ -595,6 +591,23 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return info['__files_to_merge'], info
 
+    def can_merge(self):
+        # TODO: figure out merge-capable ffmpeg version
+        if self.basename != 'avconv':
+            return True
+
+        required_version = '10-0'
+        if is_outdated_version(
+                self._versions[self.basename], required_version):
+            warning = ('Your copy of %s is outdated and unable to properly mux separate video and audio files, '
+                       'youtube-dl will download single file media. '
+                       'Update %s to version %s or newer to fix this.') % (
+                           self.basename, self.basename, required_version)
+            if self._downloader:
+                self._downloader.report_warning(warning)
+            return False
+        return True
+
 
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):

From 97fcf1bbd07ae0c5b6e530dcf2623d199452a76c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 02:01:16 +0600
Subject: [PATCH 0074/2145] [YoutubeDL] Check if merger can actually merge

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7c3bdb964..00f86b342 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1088,7 +1088,7 @@ class YoutubeDL(object):
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
                     and info_dict['extractor'] in ['youtube', 'ted']):
                 merger = FFmpegMergerPP(self)
-                if merger.available and not merger.check_outdated():
+                if merger.available and merger.can_merge():
                     req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From e41f450f2860ab5aa3f3a04bc646594c6dbc6714 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 May 2015 20:04:05 +0800
Subject: [PATCH 0075/2145] [tmz] Add support for articles (fixes #5477)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/tmz.py      | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e808f2734..b376fd279 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -543,7 +543,10 @@ from .thesixtyone import TheSixtyOneIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
-from .tmz import TMZIE
+from .tmz import (
+    TMZIE,
+    TMZArticleIE,
+)
 from .tnaflix import TNAFlixIE
 from .thvideo import (
     THVideoIE,
diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index c5c6fdc51..7dbe68b5c 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -30,3 +30,31 @@ class TMZIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'thumbnail': self._html_search_meta('ThumbURL', webpage),
         }
+
+
+class TMZArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
+    _TEST = {
+        'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
+        'md5': 'e482a414a38db73087450e3a6ce69d00',
+        'info_dict': {
+            'id': '0_6snoelag',
+            'ext': 'mp4',
+            'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
+            'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        embedded_video_info_str = self._html_search_regex(
+            r'tmzVideoEmbedV2\("([^)]+)"\);', webpage, 'embedded video info')
+
+        embedded_video_info = self._parse_json(
+            embedded_video_info_str, video_id,
+            transform_source=lambda s: s.replace('\\', ''))
+
+        return self.url_result(
+            'http://www.tmz.com/videos/%s/' % embedded_video_info['id'])

From 1f92865494c6efa1a0d5d90ffa849e85b80c8248 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 21:05:39 +0600
Subject: [PATCH 0076/2145] [dumpert] Add cpc cookie (Closes #5672)

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 9c594b757..999fb5620 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -26,7 +26,7 @@ class DumpertIE(InfoExtractor):
         video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'nsfw=1')
+        req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)
 
         files_base64 = self._search_regex(

From 511565282861e26913caddc1bcc0c865a9eec786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 21:31:36 +0600
Subject: [PATCH 0077/2145] [zingmp3] Capture error message

---
 youtube_dl/extractor/zingmp3.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 1afbe68ed..7dc1e2f2b 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -4,12 +4,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    @staticmethod
-    def _extract_item(item):
+    def _extract_item(self, item):
+        error_message = item.find('./errormessage').text
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message),
+                expected=True)
+
         title = item.find('./title').text.strip()
         source = item.find('./source').text
         extension = item.attrib['type']

From 81ed3bb9c0edb9a11b43964459ef57cca5683461 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Mon, 11 May 2015 17:38:08 +0200
Subject: [PATCH 0078/2145] [southpark] Sort alphabetically

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b376fd279..3368edf7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -489,8 +489,8 @@ from .soundgasm import (
 )
 from .southpark import (
     SouthParkIE,
-    SouthParkEsIE,
     SouthParkDeIE,
+    SouthParkEsIE,
     SouthParkNlIE
 )
 from .space import SpaceIE

From 968ee176777a1bf4e33cfb849a7241b0ac45d254 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Mon, 11 May 2015 18:02:25 +0200
Subject: [PATCH 0079/2145] [southparkdk] Add extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3368edf7c..de19dfd7a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .soundgasm import (
 from .southpark import (
     SouthParkIE,
     SouthParkDeIE,
+    SouthParkDkIE,
     SouthParkEsIE,
     SouthParkNlIE
 )
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 59e31198c..83e5a7659 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -57,3 +57,13 @@ class SouthParkNlIE(SouthParkIE):
         'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
         'playlist_count': 4,
     }]
+
+class SouthParkDkIE(SouthParkIE):
+    IE_NAME = 'southpark.dk'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'
+
+    _TESTS = [{
+        'url': 'http://www.southparkstudios.dk/full-episodes/s18e07-grounded-vindaloop',
+        'playlist_count': 4,
+    }]

From 6d3f5935e516760964052718e6b90324c6f07391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 23:47:50 +0600
Subject: [PATCH 0080/2145] [southpark] Fix IE_NAME

---
 youtube_dl/extractor/southpark.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 83e5a7659..7fb165a87 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -58,8 +58,9 @@ class SouthParkNlIE(SouthParkIE):
         'playlist_count': 4,
     }]
 
+
 class SouthParkDkIE(SouthParkIE):
-    IE_NAME = 'southpark.dk'
+    IE_NAME = 'southparkstudios.dk'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'
 

From d4b963d0a68f81f4fef5495af14e2e41add21a0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 May 2015 01:54:56 +0600
Subject: [PATCH 0081/2145] [vine] Relax `alt_title` (Closes #5677)

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 65c459fad..c733a48fa 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -75,7 +75,7 @@ class VineIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'alt_title': self._og_search_description(webpage),
+            'alt_title': self._og_search_description(webpage, default=None),
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From 5332fd91bf16867b6777bd6cfd0b5086f84112c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:42:13 +0800
Subject: [PATCH 0082/2145] [nytimes] Correct _VALID_URL of NYTimesArticleIE

---
 youtube_dl/extractor/nytimes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 6ffbe3863..7f254b867 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -89,7 +89,7 @@ class NYTimesIE(NYTimesBaseIE):
 
 
 class NYTimesArticleIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
     _TESTS = [{
         'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
         'md5': 'e2076d58b4da18e6a001d53fd56db3c9',

From 7dff03636a843a6990e52200edb3ecca1246b3df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:47:37 +0800
Subject: [PATCH 0083/2145] [utils] Support 'dur' field in TTML

---
 test/test_utils.py  | 2 +-
 youtube_dl/utils.py | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 86b110a7d..b40107037 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -600,7 +600,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                 <div xml:lang="en">
                     <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
                     <p begin="1" end="2">第二行<br/>♪♪</p>
-                    <p begin="2" end="3"><span>Third<br/>Line</span></p>
+                    <p begin="2" dur="1"><span>Third<br/>Line</span></p>
                 </div>
             </body>
             </tt>'''
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d73efcf25..5439fcb35 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1866,10 +1866,14 @@ def dfxp2srt(dfxp_data):
     paras = dfxp.findall(_x('.//ttml:p'))
 
     for para, index in zip(paras, itertools.count(1)):
+        begin_time = parse_dfxp_time_expr(para.attrib['begin'])
+        end_time = parse_dfxp_time_expr(para.attrib.get('end'))
+        if not end_time:
+            end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
-            format_srt_time(parse_dfxp_time_expr(para.attrib.get('begin'))),
-            format_srt_time(parse_dfxp_time_expr(para.attrib.get('end'))),
+            format_srt_time(begin_time),
+            format_srt_time(end_time),
             parse_node(para)))
 
     return ''.join(out)

From 1c7e2e64f6328024711d5fa999d4498396f4cb5c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:55:14 +0800
Subject: [PATCH 0084/2145] [nrk] Remove TTML to srt conversion codes

A common routine is implemented in utils.py and can be used via
--convert-subtitles.
---
 youtube_dl/extractor/nrk.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index e91d3a248..cc70c2950 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -200,20 +199,10 @@ class NRKTVIE(InfoExtractor):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
         captions = self._download_xml(
-            url, video_id, 'Downloading subtitles',
-            transform_source=lambda s: s.replace(r'<br />', '\r\n'))
+            url, video_id, 'Downloading subtitles')
         lang = captions.get('lang', 'no')
-        ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
-        srt = ''
-        for pos, p in enumerate(ps):
-            begin = parse_duration(p.get('begin'))
-            duration = parse_duration(p.get('dur'))
-            starttime = self._subtitles_timecode(begin)
-            endtime = self._subtitles_timecode(begin + duration)
-            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},
-            {'ext': 'srt', 'data': srt},
         ]}
 
     def _extract_f4m(self, manifest_url, video_id):

From c1c924abfeda45f29b991bb74f315f0e79dcf126 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 13:04:54 +0800
Subject: [PATCH 0085/2145] [utils,common] Merge format_srt_time and
 _subtitles_timecode

format_srt_time uses a comma as the delimiter between seconds and
milliseconds while _subtitles_timecode uses a dot. All .srt examples I
found on the Internet uses a comma, so I use a comma in the merged
version. See http://matroska.org/technical/specs/subtitles/srt.html and
http://devel.aegisub.org/wiki/SubtitleFormats/SRT
---
 youtube_dl/extractor/common.py    |  3 ---
 youtube_dl/extractor/kanalplay.py |  5 +++--
 youtube_dl/utils.py               | 12 ++++--------
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 981e34bc7..65bb77086 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1072,9 +1072,6 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
-    def _subtitles_timecode(self, seconds):
-        return '%02d:%02d:%02d.%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
-
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 2bb078036..4597d1b96 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
+    srt_subtitles_timecode,
 )
 
 
@@ -39,8 +40,8 @@ class KanalPlayIE(InfoExtractor):
             '%s\r\n%s --> %s\r\n%s'
             % (
                 num,
-                self._subtitles_timecode(item['startMillis'] / 1000.0),
-                self._subtitles_timecode(item['endMillis'] / 1000.0),
+                srt_subtitles_timecode(item['startMillis'] / 1000.0),
+                srt_subtitles_timecode(item['endMillis'] / 1000.0),
                 item['text'],
             ) for num, item in enumerate(subs, 1))
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5439fcb35..ed9ed9ed6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1835,12 +1835,8 @@ def parse_dfxp_time_expr(time_expr):
         return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
 
 
-def format_srt_time(seconds):
-    (mins, secs) = divmod(seconds, 60)
-    (hours, mins) = divmod(mins, 60)
-    millisecs = (secs - int(secs)) * 1000
-    secs = int(secs)
-    return '%02d:%02d:%02d,%03d' % (hours, mins, secs, millisecs)
+def srt_subtitles_timecode(seconds):
+    return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
 
 
 def dfxp2srt(dfxp_data):
@@ -1872,8 +1868,8 @@ def dfxp2srt(dfxp_data):
             end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
-            format_srt_time(begin_time),
-            format_srt_time(end_time),
+            srt_subtitles_timecode(begin_time),
+            srt_subtitles_timecode(end_time),
             parse_node(para)))
 
     return ''.join(out)

From 41333b97b9471316cf0f395db59196e6571fc776 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 12 May 2015 22:35:16 +0800
Subject: [PATCH 0086/2145] [qqmusic] Add support for charts / top lists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index de19dfd7a..8ec0c1032 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -414,6 +414,7 @@ from .qqmusic import (
     QQMusicIE,
     QQMusicSingerIE,
     QQMusicAlbumIE,
+    QQMusicToplistIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 174c8e0ae..d4a85d8c3 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    js_to_json,
 )
 from ..compat import compat_urllib_request
 
@@ -168,3 +169,57 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
             album_page, 'album details', default=None)
 
         return self.playlist_result(entries, mid, album_name, album_detail)
+
+
+class QQMusicToplistIE(QQPlaylistBaseIE):
+    _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
+    
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=toplist&p=global_12',
+        'info_dict': {
+            'id': 'global_12',
+            'title': 'itunes榜',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'http://y.qq.com/#type=toplist&p=top_6',
+        'info_dict': {
+            'id': 'top_6',
+            'title': 'QQ音乐巅峰榜·欧美',
+        },
+        'playlist_count': 100,
+    }]
+
+    @staticmethod
+    def strip_qq_jsonp(code):
+        return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
+    
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        list_type = list_id.split("_")[0]
+        num_id = list_id.split("_")[1]
+
+        list_page = self._download_webpage("http://y.qq.com/y/static/toplist/index/%s.html" % list_id, list_id, 'Download toplist page')
+        entries = []
+        if list_type == 'top':
+            list = self._download_json(
+                "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
+                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+
+            for song in list['l']:
+                s = song['s']
+                song_mid = s.split("|")[20]
+                entries.append(self.url_result(
+                    'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                    song_mid))
+
+        elif list_type == 'global':
+            entries = self.get_entries_from_page(list_page)
+
+        list_name = self._html_search_regex(
+            r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
+            default=None)
+        list_desc = None
+
+        return self.playlist_result(entries, list_id, list_name, list_desc)
\ No newline at end of file

From b480e7874b45862eae343ab8484aa43381cd28fa Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 12 May 2015 22:41:37 +0800
Subject: [PATCH 0087/2145] [qqmusic] Fix code formatting

---
 youtube_dl/extractor/qqmusic.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index d4a85d8c3..bca4a8f90 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -200,12 +200,15 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_type = list_id.split("_")[0]
         num_id = list_id.split("_")[1]
 
-        list_page = self._download_webpage("http://y.qq.com/y/static/toplist/index/%s.html" % list_id, list_id, 'Download toplist page')
+        list_page = self._download_webpage(
+            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
+            list_id, 'Download toplist page')
         entries = []
         if list_type == 'top':
             list = self._download_json(
                 "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
-                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', 
+                transform_source=self.strip_qq_jsonp)
 
             for song in list['l']:
                 s = song['s']
@@ -222,4 +225,5 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             default=None)
         list_desc = None
 
-        return self.playlist_result(entries, list_id, list_name, list_desc)
\ No newline at end of file
+        return self.playlist_result(entries, list_id, list_name, list_desc)
+        
\ No newline at end of file

From fd4eefed39595850b864d3be9711224e4e8e9dd4 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 13 May 2015 01:14:02 +0800
Subject: [PATCH 0088/2145] [qqmusic] Fix extraction for global list

---
 youtube_dl/extractor/qqmusic.py | 34 ++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bca4a8f90..3401dcaef 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -188,6 +188,13 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             'title': 'QQ音乐巅峰榜·欧美',
         },
         'playlist_count': 100,
+    }, {
+        'url': 'http://y.qq.com/#type=toplist&p=global_5',
+        'info_dict': {
+            'id': 'global_5',
+            'title': '韩国mnet排行榜',
+        },
+        'playlist_count': 50,
     }]
 
     @staticmethod
@@ -203,22 +210,23 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_page = self._download_webpage(
             "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
             list_id, 'Download toplist page')
+
         entries = []
+        jsonp_url = ""
         if list_type == 'top':
-            list = self._download_json(
-                "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
-                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', 
-                transform_source=self.strip_qq_jsonp)
-
-            for song in list['l']:
-                s = song['s']
-                song_mid = s.split("|")[20]
-                entries.append(self.url_result(
-                    'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
-                    song_mid))
-
+            jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
         elif list_type == 'global':
-            entries = self.get_entries_from_page(list_page)
+            jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
+        
+        list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 
+            errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+            
+        for song in list['l']:
+            s = song['s']
+            song_mid = s.split("|")[20]
+            entries.append(self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                song_mid))
 
         list_name = self._html_search_regex(
             r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',

From 86ec1e487c4908f4d0d0ece512007a2e5fedc593 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 13 May 2015 01:37:56 +0800
Subject: [PATCH 0089/2145] [qqmusic] Code fixes

---
 youtube_dl/extractor/qqmusic.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 3401dcaef..bae2ce31a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -212,10 +212,9 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             list_id, 'Download toplist page')
 
         entries = []
-        jsonp_url = ""
         if list_type == 'top':
             jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
-        elif list_type == 'global':
+        else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
         
         list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 

From 0b4253fa3710c656e12b6147ed7c1f7843bb9aae Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Tue, 12 May 2015 18:57:06 +0100
Subject: [PATCH 0090/2145] [BYUtvIE] Change thumbnail regex so test does not
 fail

---
 youtube_dl/extractor/byutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 6252be05b..3b2de517e 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -16,7 +16,7 @@ class BYUtvIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:5438d33774b6bdc662f9485a340401cc',
             'title': 'Season 5 Episode 5',
-            'thumbnail': 're:^https?://.*promo.*'
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
         'params': {
             'skip_download': True,

From 3749e36e9f0e6be2a3a3ab1b15c0c02be5a50e2f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 May 2015 21:16:45 +0800
Subject: [PATCH 0091/2145] [YoutubeDL] Fix PEP8 W503

---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cf83c510..84d50dab7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1085,9 +1085,9 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
-                    and info_dict['extractor'] in ['youtube', 'ted']
-                    and FFmpegMergerPP(self).available):
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
+                    info_dict['extractor'] in ['youtube', 'ted'] and
+                    FFmpegMergerPP(self).available):
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 372744c544ec3de1b35583e7d6fc2cbc4cc39f3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 May 2015 22:26:30 +0600
Subject: [PATCH 0092/2145] [odnoklassniki] Fix extraction (Closes #5671)

---
 youtube_dl/extractor/odnoklassniki.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 155d0ee6a..fbc521d1a 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -6,6 +6,7 @@ from ..utils import (
     unified_strdate,
     int_or_none,
     qualities,
+    unescapeHTML,
 )
 
 
@@ -36,8 +37,8 @@ class OdnoklassnikiIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         player = self._parse_json(
-            self._search_regex(
-                r"OKVideo\.start\(({.+?})\s*,\s*'VideoAutoplay_player'", webpage, 'player'),
+            unescapeHTML(self._search_regex(
+                r'data-attributes="([^"]+)"', webpage, 'player')),
             video_id)
 
         metadata = self._parse_json(player['flashvars']['metadata'], video_id)

From 8e595397529abc71093264e3695fb00d95be4d78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 02:32:00 +0800
Subject: [PATCH 0093/2145] [postprocessor/embedthumbnail] Use thumbnails
 downloaded by YoutubeDL

---
 youtube_dl/YoutubeDL.py                    |  2 +-
 youtube_dl/__init__.py                     |  8 +++++++-
 youtube_dl/postprocessor/embedthumbnail.py | 23 +++++++++++-----------
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 84d50dab7..0fbfe9642 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1848,7 +1848,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
-            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
 
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 9cc9f851f..ace17857c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -240,7 +240,13 @@ def _real_main(argv=None):
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
     if opts.embedthumbnail:
-        postprocessors.append({'key': 'EmbedThumbnail'})
+        already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
+        postprocessors.append({
+            'key': 'EmbedThumbnail',
+            'already_have_thumbnail': already_have_thumbnail
+        })
+        if not already_have_thumbnail:
+            opts.writethumbnail = True
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 4868a42fd..95c52f65f 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -7,11 +7,7 @@ import subprocess
 
 from .ffmpeg import FFmpegPostProcessor
 
-from ..compat import (
-    compat_urlretrieve,
-)
 from ..utils import (
-    determine_ext,
     check_executable,
     encodeFilename,
     PostProcessingError,
@@ -25,26 +21,30 @@ class EmbedThumbnailPPError(PostProcessingError):
 
 
 class EmbedThumbnailPP(FFmpegPostProcessor):
+    def __init__(self, downloader=None, already_have_thumbnail=False):
+        super(EmbedThumbnailPP, self).__init__(downloader)
+        self._already_have_thumbnail = already_have_thumbnail
+
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = filename + '.' + determine_ext(info['thumbnail'])
 
-        if not info.get('thumbnail'):
+        if not info.get('thumbnails'):
             raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')
 
-        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
+        thumbnail_filename = info['thumbnails'][-1]['filename']
 
         if info['ext'] == 'mp3':
             options = [
-                '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+                '-i', thumbnail_filename, '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
             self.run_ffmpeg(filename, temp_filename, options)
 
-            os.remove(encodeFilename(temp_thumbnail))
+            if not self._already_have_thumbnail:
+                os.remove(encodeFilename(thumbnail_filename))
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
@@ -52,7 +52,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
+            cmd = ['AtomicParsley', filename, '--artwork', thumbnail_filename, '-o', temp_filename]
 
             self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
 
@@ -66,7 +66,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 msg = stderr.decode('utf-8', 'replace').strip()
                 raise EmbedThumbnailPPError(msg)
 
-            os.remove(encodeFilename(temp_thumbnail))
+            if not self._already_have_thumbnail:
+                os.remove(encodeFilename(thumbnail_filename))
             # for formats that don't support thumbnails (like 3gp) AtomicParsley
             # won't create to the temporary file
             if b'No changes' in stdout:

From bb8ca1d112e95cd3fe48fff5af980a62a9db2572 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 02:35:28 +0800
Subject: [PATCH 0094/2145] [postprocessor/embedthumbnail] Use
 run_ffmpeg_multiple_files

---
 youtube_dl/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 95c52f65f..4e08c2709 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -36,12 +36,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
 
         if info['ext'] == 'mp3':
             options = [
-                '-i', thumbnail_filename, '-c', 'copy', '-map', '0', '-map', '1',
+                '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
-            self.run_ffmpeg(filename, temp_filename, options)
+            self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options)
 
             if not self._already_have_thumbnail:
                 os.remove(encodeFilename(thumbnail_filename))

From 2cc6d135479c5dbd6e715a1e767c5be163cd22ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 04:41:30 +0800
Subject: [PATCH 0095/2145] [postprocessor/embedthumbnail] Encode arguments in
 calling AtomicParsley

---
 youtube_dl/postprocessor/embedthumbnail.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 4e08c2709..8f825f785 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -9,6 +9,7 @@ from .ffmpeg import FFmpegPostProcessor
 
 from ..utils import (
     check_executable,
+    encodeArgument,
     encodeFilename,
     PostProcessingError,
     prepend_extension,
@@ -52,7 +53,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = ['AtomicParsley', filename, '--artwork', thumbnail_filename, '-o', temp_filename]
+            cmd = [encodeFilename('AtomicParsley', True),
+                   encodeFilename(filename, True),
+                   encodeArgument('--artwork'),
+                   encodeFilename(thumbnail_filename, True),
+                   encodeArgument('-o'),
+                   encodeFilename(temp_filename, True)]
 
             self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
 

From 86c7fdb17c0dcbff88a8daa131fddc57b6304b83 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 14:26:47 +0800
Subject: [PATCH 0096/2145] [xattr] Enhance error handling to catch ENOSPC

Fixes #5589
---
 youtube_dl/postprocessor/xattrpp.py | 61 +++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 93d0abcf6..16f2966e9 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -3,18 +3,32 @@ from __future__ import unicode_literals
 import os
 import subprocess
 import sys
+import errno
 
 from .common import PostProcessor
-from ..compat import (
-    subprocess_check_output
-)
 from ..utils import (
     check_executable,
     hyphenate_date,
     version_tuple,
+    PostProcessingError,
+    encodeArgument,
+    encodeFilename,
 )
 
 
+class XAttrMetadataError(PostProcessingError):
+    def __init__(self, code=None, msg='Unknown error'):
+        super(XAttrMetadataError, self).__init__(msg)
+        self.code = code
+
+        # Parsing code and msg
+        if (self.code in (errno.ENOSPC, errno.EDQUOT) or
+                'No space left' in self.msg or 'Disk quota excedded' in self.msg):
+            self.reason = 'NO_SPACE'
+        else:
+            self.reason = 'NOT_SUPPORTED'
+
+
 class XAttrMetadataPP(PostProcessor):
 
     #
@@ -51,7 +65,10 @@ class XAttrMetadataPP(PostProcessor):
                 raise ImportError
 
             def write_xattr(path, key, value):
-                return xattr.setxattr(path, key, value)
+                try:
+                    xattr.set(path, key, value)
+                except EnvironmentError as e:
+                    raise XAttrMetadataError(e.errno, e.strerror)
 
         except ImportError:
             if os.name == 'nt':
@@ -62,8 +79,11 @@ class XAttrMetadataPP(PostProcessor):
                     assert os.path.exists(path)
 
                     ads_fn = path + ":" + key
-                    with open(ads_fn, "wb") as f:
-                        f.write(value)
+                    try:
+                        with open(ads_fn, "wb") as f:
+                            f.write(value)
+                    except EnvironmentError as e:
+                        raise XAttrMetadataError(e.errno, e.strerror)
             else:
                 user_has_setfattr = check_executable("setfattr", ['--version'])
                 user_has_xattr = check_executable("xattr", ['-h'])
@@ -71,12 +91,24 @@ class XAttrMetadataPP(PostProcessor):
                 if user_has_setfattr or user_has_xattr:
 
                     def write_xattr(path, key, value):
+                        value = value.decode('utf-8')
                         if user_has_setfattr:
-                            cmd = ['setfattr', '-n', key, '-v', value, path]
+                            executable = 'setfattr'
+                            opts = ['-n', key, '-v', value]
                         elif user_has_xattr:
-                            cmd = ['xattr', '-w', key, value, path]
+                            executable = 'xattr'
+                            opts = ['-w', key, value]
 
-                        subprocess_check_output(cmd)
+                        cmd = ([encodeFilename(executable, True)] +
+                               [encodeArgument(o) for o in opts] +
+                               [encodeFilename(path, True)])
+
+                        p = subprocess.Popen(
+                            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        stdout, stderr = p.communicate()
+                        stderr = stderr.decode('utf-8', 'replace')
+                        if p.returncode != 0:
+                            raise XAttrMetadataError(p.returncode, stderr)
 
                 else:
                     # On Unix, and can't find pyxattr, setfattr, or xattr.
@@ -121,6 +153,13 @@ class XAttrMetadataPP(PostProcessor):
 
             return [], info
 
-        except (subprocess.CalledProcessError, OSError):
-            self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
+        except XAttrMetadataError as e:
+            if e.reason == 'NO_SPACE':
+                self._downloader.report_warning(
+                    'There\'s no disk space left or disk quota exceeded. ' +
+                    'Extended attributes are not written.')
+            else:
+                self._downloader.report_error(
+                    'This filesystem doesn\'t support extended attributes. ' +
+                    '(You may have to enable them in your /etc/fstab)')
             return [], info

From fbff30d2dbc6462c628384ea5960c2461e7cdcca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 14:51:00 +0800
Subject: [PATCH 0097/2145] [xattr] Catch 'Argument list too long'

---
 youtube_dl/postprocessor/xattrpp.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 16f2966e9..27e273000 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -25,6 +25,8 @@ class XAttrMetadataError(PostProcessingError):
         if (self.code in (errno.ENOSPC, errno.EDQUOT) or
                 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
             self.reason = 'NO_SPACE'
+        elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
+            self.reason = 'VALUE_TOO_LONG'
         else:
             self.reason = 'NOT_SUPPORTED'
 
@@ -103,8 +105,11 @@ class XAttrMetadataPP(PostProcessor):
                                [encodeArgument(o) for o in opts] +
                                [encodeFilename(path, True)])
 
-                        p = subprocess.Popen(
-                            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        try:
+                            p = subprocess.Popen(
+                                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        except EnvironmentError as e:
+                            raise XAttrMetadataError(e.errno, e.strerror)
                         stdout, stderr = p.communicate()
                         stderr = stderr.decode('utf-8', 'replace')
                         if p.returncode != 0:
@@ -158,6 +163,9 @@ class XAttrMetadataPP(PostProcessor):
                 self._downloader.report_warning(
                     'There\'s no disk space left or disk quota exceeded. ' +
                     'Extended attributes are not written.')
+            elif e.reason == 'VALUE_TOO_LONG':
+                self._downloader.report_warning(
+                    'Unable to write extended attributes due to too long values.')
             else:
                 self._downloader.report_error(
                     'This filesystem doesn\'t support extended attributes. ' +

From 509c630db8cdaff473f95805cda1ae350107e36b Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 14 May 2015 08:09:56 +0100
Subject: [PATCH 0098/2145] [CanalplusIE] Update tests that were no longer
 working

---
 youtube_dl/extractor/canalplus.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 1b14471e5..71801488a 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -25,14 +25,13 @@ class CanalplusIE(InfoExtractor):
     }
 
     _TESTS = [{
-        'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
-        'md5': '3db39fb48b9685438ecf33a1078023e4',
+        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
         'info_dict': {
-            'id': '922470',
+            'id': '1263092',
             'ext': 'flv',
-            'title': 'Zapping - 26/08/13',
-            'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
-            'upload_date': '20130826',
+            'title': 'Le Zapping - 13/05/15',
+            'description': 'md5:09738c0d06be4b5d06a0940edb0da73f',
+            'upload_date': '20150513',
         },
     }, {
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
@@ -56,7 +55,6 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
-        'md5': '65aa83ad62fe107ce29e564bb8712580',
         'info_dict': {
             'id': '1213714',
             'ext': 'flv',

From c827d4cfdb9ce47d13ccbec32d2b32dfb429ea8a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 16:53:10 +0800
Subject: [PATCH 0099/2145] [xattr] Enhanced error messages on Windows

---
 youtube_dl/postprocessor/xattrpp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 27e273000..7d88e1308 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -167,7 +167,10 @@ class XAttrMetadataPP(PostProcessor):
                 self._downloader.report_warning(
                     'Unable to write extended attributes due to too long values.')
             else:
-                self._downloader.report_error(
-                    'This filesystem doesn\'t support extended attributes. ' +
-                    '(You may have to enable them in your /etc/fstab)')
+                msg = 'This filesystem doesn\'t support extended attributes. '
+                if os.name == 'nt':
+                    msg += 'You need to use NTFS.'
+                else:
+                    msg += '(You may have to enable them in your /etc/fstab)'
+                self._downloader.report_error(msg)
             return [], info

From 7d57d2e18be416faa593364966ccf667243fd3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 14:59:27 +0600
Subject: [PATCH 0100/2145] [canalplus] Restore checksums in tests

---
 youtube_dl/extractor/canalplus.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 71801488a..699b4f7d0 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -26,6 +26,7 @@ class CanalplusIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
+        'md5': 'b3481d7ca972f61e37420798d0a9d934',
         'info_dict': {
             'id': '1263092',
             'ext': 'flv',
@@ -55,6 +56,7 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
+        'md5': 'f3a46edcdf28006598ffaf5b30e6a2d4',
         'info_dict': {
             'id': '1213714',
             'ext': 'flv',

From 82245a6de77f4755b063310258c5611c15f5ffbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 15:21:27 +0600
Subject: [PATCH 0101/2145] [YoutubeDL] Restore filename for thumbnails

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c8196d08..691f3e09f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1849,7 +1849,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
-            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
 
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %

From fa6a16996e4a1aeee4e421b172efc6c351b1b123 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 18:00:57 +0800
Subject: [PATCH 0102/2145] [worldstarhiphop] Support Android URLs (fixes
 #5629)

---
 youtube_dl/extractor/worldstarhiphop.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index d5c26a032..a3ea26feb 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 
 
 class WorldStarHipHopIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/videos/video\.php\?v=(?P<id>.*)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
+    _TESTS = [{
         "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
         "md5": "9d04de741161603bf7071bbf4e883186",
         "info_dict": {
@@ -15,7 +15,15 @@ class WorldStarHipHopIE(InfoExtractor):
             "ext": "mp4",
             "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
         }
-    }
+    }, {
+        'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
+        'md5': 'dc1c76c83ecc4190bb1eb143899b87d3',
+        'info_dict': {
+            'id': 'wshh6a7q1ny0G34ZwuIO',
+            'ext': 'mp4',
+            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -26,19 +34,22 @@ class WorldStarHipHopIE(InfoExtractor):
             return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')
 
         video_url = self._search_regex(
-            r'so\.addVariable\("file","(.*?)"\)', webpage, 'video URL')
+            [r'so\.addVariable\("file","(.*?)"\)',
+             r'<div class="artlist">\s*<a[^>]+href="([^"]+)">'],
+            webpage, 'video URL')
 
         if 'youtube' in video_url:
             return self.url_result(video_url, ie='Youtube')
 
         video_title = self._html_search_regex(
-            r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+            [r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+             r'<span[^>]+class="tc-sp-pinned-title">(.*)</span>'],
             webpage, 'title')
 
         # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
         thumbnail = self._html_search_regex(
             r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
-            fatal=False)
+            default=None)
         if not thumbnail:
             _title = r'candytitles.*>(.*)</span>'
             mobj = re.search(_title, webpage)

From 7a012d5a16632a103466f9e9794dd98ad573ce88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 16:39:35 +0600
Subject: [PATCH 0103/2145] [screenwavemedia] Add support for player2 URLs
 (Closes #5696)

---
 youtube_dl/extractor/screenwavemedia.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 74fb1983a..d1ab66b32 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
+    _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
@@ -20,7 +20,10 @@ class ScreenwaveMediaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
+
+        playerdata = self._download_webpage(
+            'http://player.screenwavemedia.com/play/player.php?id=%s' % video_id,
+            video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
             r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
@@ -99,7 +102,7 @@ class TeamFourIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            r'src="(http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
             webpage, 'player data URL')
 
         video_title = self._html_search_regex(

From 548897396158d7822020f45c10301e9ca3c46453 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:25:43 +0800
Subject: [PATCH 0104/2145] [qqmusic] flake8

---
 youtube_dl/extractor/qqmusic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bae2ce31a..5ddbb183e 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -173,7 +173,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
-    
+
     _TESTS = [{
         'url': 'http://y.qq.com/#type=toplist&p=global_12',
         'info_dict': {
@@ -200,7 +200,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     @staticmethod
     def strip_qq_jsonp(code):
         return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
-    
+
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
@@ -208,7 +208,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         num_id = list_id.split("_")[1]
 
         list_page = self._download_webpage(
-            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
+            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
             list_id, 'Download toplist page')
 
         entries = []
@@ -216,10 +216,11 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
         else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
-        
-        list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 
+
+        list = self._download_json(
+            jsonp_url, list_id, note='Retrieve toplist json',
             errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
-            
+
         for song in list['l']:
             s = song['s']
             song_mid = s.split("|")[20]
@@ -233,4 +234,3 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_desc = None
 
         return self.playlist_result(entries, list_id, list_name, list_desc)
-        
\ No newline at end of file

From 29ea57283e473e94c72cf9cbc065c3c05a14830f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:28:42 +0800
Subject: [PATCH 0105/2145] [qqmusic] Refactoring QQMusicToplistIE

---
 youtube_dl/extractor/qqmusic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 5ddbb183e..59e93a1dd 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -204,8 +204,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        list_type = list_id.split("_")[0]
-        num_id = list_id.split("_")[1]
+        list_type, num_id = list_id.split("_")
 
         list_page = self._download_webpage(
             "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
@@ -217,11 +216,11 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
 
-        list = self._download_json(
+        toplist_json = self._download_json(
             jsonp_url, list_id, note='Retrieve toplist json',
             errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
 
-        for song in list['l']:
+        for song in toplist_json['l']:
             s = song['s']
             song_mid = s.split("|")[20]
             entries.append(self.url_result(
@@ -231,6 +230,5 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_name = self._html_search_regex(
             r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
             default=None)
-        list_desc = None
 
-        return self.playlist_result(entries, list_id, list_name, list_desc)
+        return self.playlist_result(entries, list_id, list_name)

From 7ec676bb3dd6cba4b56fccb2d5aae08e66086b4e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:32:36 +0800
Subject: [PATCH 0106/2145] [qqmusic] Add IE_NAME for all extractors

---
 youtube_dl/extractor/qqmusic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 59e93a1dd..13113820b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -15,6 +15,7 @@ from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
+    IE_NAME = 'qqmusic'
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
@@ -97,6 +98,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 
 
 class QQMusicSingerIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:singer'
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -140,6 +142,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
 
 class QQMusicAlbumIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:album'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TEST = {
@@ -172,6 +175,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:toplist'
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{

From 1ae72fb23df709687091133602fd715ab6cb7b2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 22:28:42 +0600
Subject: [PATCH 0107/2145] [soundcloud:user] Defer download link resolve
 (Closes #5248)

Looks like final download links can expire before downloading process reach them. So, resolving download links right before actual downloading.
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 183ff50f4..c23c5ee0f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -336,7 +336,7 @@ class SoundcloudUserIE(SoundcloudIE):
             if len(new_entries) == 0:
                 self.to_screen('%s: End page received' % uploader)
                 break
-            entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
+            entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries)
 
         return {
             '_type': 'playlist',

From 3a105f7b20e8a3f742ac86cc1a6b02935b831778 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 May 2015 02:17:22 +0800
Subject: [PATCH 0108/2145] [teamcoco] Rewrite preload data extraction

Idea: "puncture" some consecutive fragments and check whether the
b64decode result of a punctured string is a valid JSON or not.

It's a O(N^3) algorithm, but should be fast for a small N (less than 30
fragments in all test cases)
---
 youtube_dl/extractor/teamcoco.py | 53 ++++++++++++++++----------------
 1 file changed, 26 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 95d58ddd0..410eb7d3a 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import base64
 import binascii
 import re
+import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -68,41 +69,39 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        data = preload = None
-        preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
-        if preloads:
-            preload = max([(len(p), p) for p in preloads])[1]
+        data = None
 
-        if not preload:
-            preload = ''.join(re.findall(r'this\.push\("([^"]+)"\);', webpage))
+        preload_codes = self._html_search_regex(
+            r'(function.+)setTimeout\(function\(\)\{playlist',
+            webpage, 'preload codes')
+        base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+        base64_fragments.remove('init')
 
-        if not preload:
-            preload = self._html_search_regex([
-                r'player,\[?"([^"]+)"\]?', r'player.init\(\[?"([^"]+)"\]?\)'
-            ], webpage.replace('","', ''), 'preload data', default=None)
-
-        if not preload:
-            preload_codes = self._html_search_regex(
-                r'(function.+)setTimeout\(function\(\)\{playlist',
-                webpage, 'preload codes')
-            base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
-            base64_fragments.remove('init')
-            for i in range(len(base64_fragments)):
-                cur_sequence = (''.join(base64_fragments[i:] + base64_fragments[:i])).encode('ascii')
+        def _check_sequence(cur_fragments):
+            if not cur_fragments:
+                return
+            for i in range(len(cur_fragments)):
+                cur_sequence = (''.join(cur_fragments[i:] + cur_fragments[:i])).encode('ascii')
                 try:
                     raw_data = base64.b64decode(cur_sequence)
-                except (TypeError, binascii.Error):
+                    if compat_ord(raw_data[0]) == compat_ord('{'):
+                        return json.loads(raw_data.decode('utf-8'))
+                except (TypeError, binascii.Error, UnicodeDecodeError, ValueError):
                     continue
-                if compat_ord(raw_data[0]) == compat_ord('{'):
-                    data = self._parse_json(raw_data.decode('utf-8'), video_id, fatal=False)
 
-        if not preload and not data:
-            raise ExtractorError(
-                'Preload information could not be extracted', expected=True)
+        def _check_data():
+            for i in range(len(base64_fragments) + 1):
+                for j in range(i, len(base64_fragments) + 1):
+                    data = _check_sequence(base64_fragments[:i] + base64_fragments[j:])
+                    if data:
+                        return data
+
+        self.to_screen('Try to compute possible data sequence. This may take some time.')
+        data = _check_data()
 
         if not data:
-            data = self._parse_json(
-                base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
+            raise ExtractorError(
+                'Preload information could not be extracted', expected=True)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From 12675275a1d2158fbe409361888569e4cb52ef07 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 May 2015 02:27:41 +0800
Subject: [PATCH 0109/2145] [teamcoco] Detect expired videos (#5626)

---
 youtube_dl/extractor/teamcoco.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 410eb7d3a..56be52638 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -62,7 +62,9 @@ class TeamcocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id)
+        webpage, urlh = self._download_webpage_handle(url, display_id)
+        if 'src=expired' in urlh.geturl():
+            raise ExtractorError('This video is expired.', expected=True)
 
         video_id = mobj.group('video_id')
         if not video_id:

From 2bc43303031215436b201e656094b60ab3ec7e9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 May 2015 23:41:27 +0200
Subject: [PATCH 0110/2145] [youtube:history] Fix extraction (fixes #5702)

It uses the same method as YoutubeSubscriptionsIE, if other feed starts using it we should consider using base class.
---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0869c9fd4..e58184adc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1667,13 +1667,42 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
         return self._extract_playlist('WL')
 
 
-class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+class YoutubeHistoryIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:history'
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
-    _FEED_NAME = 'history'
-    _PERSONAL_FEED = True
-    _PLAYLIST_TITLE = 'Youtube Watch History'
+    _TESTS = []
+
+    def _real_extract(self, url):
+        title = 'Youtube History'
+        page = self._download_webpage('https://www.youtube.com/feed/history', title)
+
+        # The extraction process is the same as for playlists, but the regex
+        # for the video ids doesn't contain an index
+        ids = []
+        more_widget_html = content_html = page
+
+        for page_num in itertools.count(1):
+            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
+            new_ids = orderedSet(matches)
+            ids.extend(new_ids)
+
+            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+            if not mobj:
+                break
+
+            more = self._download_json(
+                'https://youtube.com/%s' % mobj.group('more'), title,
+                'Downloading page #%s' % page_num,
+                transform_source=uppercase_escape)
+            content_html = more['content_html']
+            more_widget_html = more['load_more_widget_html']
+
+        return {
+            '_type': 'playlist',
+            'title': title,
+            'entries': self._ids_to_results(ids),
+        }
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):

From c4fc559f45ea5c40409eab44867ff2b4f08976c2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 May 2015 10:13:43 +0200
Subject: [PATCH 0111/2145] release 2015.05.15

---
 docs/supportedsites.md | 11 ++++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 98b625380..43fbe8b1d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -291,6 +291,7 @@
  - **MySpass**
  - **myvideo**
  - **MyVidster**
+ - **N-JOY**
  - **n-tv.de**
  - **NationalGeographic**
  - **Naver**
@@ -368,9 +369,10 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
- - **QQMusic**
- - **QQMusicAlbum**
- - **QQMusicSinger**
+ - **qqmusic**
+ - **qqmusic:album**
+ - **qqmusic:singer**
+ - **qqmusic:toplist**
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -439,6 +441,7 @@
  - **southpark.cc.com:español**
  - **southpark.de**
  - **southpark.nl**
+ - **southparkstudios.dk**
  - **Space**
  - **SpankBang**
  - **Spankwire**
@@ -492,6 +495,7 @@
  - **tlc.com**
  - **tlc.de**
  - **TMZ**
+ - **TMZArticle**
  - **TNAFlix**
  - **tou.tv**
  - **Toypics**: Toypics user profile
@@ -569,6 +573,7 @@
  - **vk.com**
  - **vk.com:user-videos**: vk.com:All of a user's videos
  - **Vodlocker**
+ - **VoiceRepublic**
  - **Vporn**
  - **VRT**
  - **vube**: Vube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 83c5a1659..38f00bc9b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.10'
+__version__ = '2015.05.15'

From 3884dcf313223040049e4153e0c398fbc36b5117 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 14:03:00 +0200
Subject: [PATCH 0112/2145] YoutubeDL: ignore indexes from 'playlist_items'
 that are not in the list (fixes #5706)

We ignore them instead of failing to match the behaviour of the 'playliststart' parameter.
---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 691f3e09f..5df889945 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -759,7 +759,9 @@ class YoutubeDL(object):
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
                 if playlistitems:
-                    entries = [ie_entries[i - 1] for i in playlistitems]
+                    entries = [
+                        ie_entries[i - 1] for i in playlistitems
+                        if -n_all_entries <= i - 1 < n_all_entries]
                 else:
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)

From e9eaf3fbcf497e76a55d2ba15d5880af83a065d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 14:06:19 +0200
Subject: [PATCH 0113/2145] [test/YoutubeDL] Add tests for 'playliststart',
 'playlistend' and 'playlist_items'

---
 test/test_YoutubeDL.py | 46 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 82b827536..a13c09ef4 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -12,6 +12,7 @@ import copy
 
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_str
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import match_filter_func
@@ -507,6 +508,51 @@ class TestYoutubeDL(unittest.TestCase):
         res = get_videos(f)
         self.assertEqual(res, ['1'])
 
+    def test_playlist_items_selection(self):
+        entries = [{
+            'id': compat_str(i),
+            'title': compat_str(i),
+            'url': TEST_URL,
+        } for i in range(1, 5)]
+        playlist = {
+            '_type': 'playlist',
+            'id': 'test',
+            'entries': entries,
+            'extractor': 'test:playlist',
+            'extractor_key': 'test:playlist',
+            'webpage_url': 'http://example.com',
+        }
+
+        def get_ids(params):
+            ydl = YDL(params)
+            # make a copy because the dictionary can be modified
+            ydl.process_ie_result(playlist.copy())
+            return [int(v['id']) for v in ydl.downloaded_info_dicts]
+
+        result = get_ids({})
+        self.assertEqual(result, [1, 2, 3, 4])
+
+        result = get_ids({'playlistend': 10})
+        self.assertEqual(result, [1, 2, 3, 4])
+
+        result = get_ids({'playlistend': 2})
+        self.assertEqual(result, [1, 2])
+
+        result = get_ids({'playliststart': 10})
+        self.assertEqual(result, [])
+
+        result = get_ids({'playliststart': 2})
+        self.assertEqual(result, [2, 3, 4])
+
+        result = get_ids({'playlist_items': '2-4'})
+        self.assertEqual(result, [2, 3, 4])
+
+        result = get_ids({'playlist_items': '2,4'})
+        self.assertEqual(result, [2, 4])
+
+        result = get_ids({'playlist_items': '10'})
+        self.assertEqual(result, [])
+
 
 if __name__ == '__main__':
     unittest.main()

From 15da7ce7fb89203247f4c959a748281ecf353e2a Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 15 May 2015 12:28:10 +0100
Subject: [PATCH 0114/2145] Fix file format extraction regex and update test
 file checksum

---
 youtube_dl/extractor/ccc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 2a5d4be18..6924eac70 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -16,7 +16,7 @@ class CCCIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
-        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'md5': '3a1eda8f3a29515d27f5adb967d7e740',
         'info_dict': {
             'id': '20131228183',
             'ext': 'mp4',
@@ -51,7 +51,7 @@ class CCCIE(InfoExtractor):
 
         matches = re.finditer(r'''(?xs)
             <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
-            <a\s+href='(?P<http_url>[^']+)'>\s*
+            <a\s+download\s+href='(?P<http_url>[^']+)'>\s*
             (?:
                 .*?
                 <a\s+href='(?P<torrent_url>[^']+\.torrent)'

From a7b8467ac0baecd02a815b1f57731ae9bb10ab87 Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 16:52:11 +0400
Subject: [PATCH 0115/2145] Sportbox extractor fix.

---
 youtube_dl/extractor/sportbox.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index becdf658f..830220543 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SportBoxIE(InfoExtractor):
-    _VALID_URL = r'https?://news\.sportbox\.ru/Vidy_sporta/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
+    _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
     _TESTS = [
         {
             'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
@@ -50,7 +50,7 @@ class SportBoxIE(InfoExtractor):
             display_id, 'Downloading player webpage')
 
         hls = self._search_regex(
-            r"var\s+original_hls_file\s*=\s*'([^']+)'", player, 'hls file')
+            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]+([^\"]+)['\"]+", player, 'hls file')
 
         formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
 

From ae670a6ed8019f1b69bbe345621f51c8b32789ec Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 17:53:05 +0400
Subject: [PATCH 0116/2145] Sportbox source fix. HD videos support.

---
 youtube_dl/extractor/sportbox.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 830220543..695b3ff82 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -14,7 +14,7 @@ class SportBoxIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
     _TESTS = [
         {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',            
             'md5': 'ff56a598c2cf411a9a38a69709e97079',
             'info_dict': {
                 'id': '80822',
@@ -42,11 +42,15 @@ class SportBoxIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._search_regex(
-            r'src="/vdl/player/media/(\d+)"', webpage, 'video id')
+        sobj = re.search(r'src="/vdl/player/(?P<media_type>\w+)/(?P<video_id>\d+)"', webpage)
+        if (sobj):
+            video_id = sobj.group('video_id')
+            media_type = sobj.group('media_type')
+        else:
+            raise RegexNotFoundError('Unable to extract video_id')
 
         player = self._download_webpage(
-            'http://news.sportbox.ru/vdl/player/media/%s' % video_id,
+            'http://news.sportbox.ru/vdl/player/%s/%s' % (media_type, video_id),
             display_id, 'Downloading player webpage')
 
         hls = self._search_regex(

From 25f14e9f93295a787e0cb436a5f6179d6174733d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 21:06:59 +0600
Subject: [PATCH 0117/2145] [youtube] Separate feed extractor

---
 youtube_dl/extractor/youtube.py | 143 +++++++++-----------------------
 1 file changed, 37 insertions(+), 106 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e58184adc..9096a2975 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -49,6 +49,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             # YouTube sets the expire time to about two months
             expire_time=time.time() + 2 * 30 * 24 * 3600)
 
+    def _ids_to_results(self, ids):
+        return [
+            self.url_result(vid_id, 'Youtube', video_id=vid_id)
+            for vid_id in ids]
+
     def _login(self):
         """
         Attempt to log in to YouTube.
@@ -1261,11 +1266,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _ids_to_results(self, ids):
-        return [
-            self.url_result(vid_id, 'Youtube', video_id=vid_id)
-            for vid_id in ids]
-
     def _extract_mix(self, playlist_id):
         # The mixes are generated from a single video
         # the id of the playlist is just 'RD' + video_id
@@ -1601,20 +1601,10 @@ class YoutubeShowIE(InfoExtractor):
 
 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
     """
-    Base class for extractors that fetch info from
-    http://www.youtube.com/feed_ajax
+    Base class for feed extractors
     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
     """
     _LOGIN_REQUIRED = True
-    # use action_load_personal_feed instead of action_load_system_feed
-    _PERSONAL_FEED = False
-
-    @property
-    def _FEED_TEMPLATE(self):
-        action = 'action_load_system_feed'
-        if self._PERSONAL_FEED:
-            action = 'action_load_personal_feed'
-        return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
 
     @property
     def IE_NAME(self):
@@ -1624,58 +1614,8 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        feed_entries = []
-        paging = 0
-        for i in itertools.count(1):
-            info = self._download_json(
-                self._FEED_TEMPLATE % paging,
-                '%s feed' % self._FEED_NAME,
-                'Downloading page %s' % i,
-                transform_source=uppercase_escape)
-            feed_html = info.get('feed_html') or info.get('content_html')
-            load_more_widget_html = info.get('load_more_widget_html') or feed_html
-            m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
-            ids = orderedSet(m.group(1) for m in m_ids)
-            feed_entries.extend(
-                self.url_result(video_id, 'Youtube', video_id=video_id)
-                for video_id in ids)
-            mobj = re.search(
-                r'data-uix-load-more-href="/?[^"]+paging=(?P<paging>\d+)',
-                load_more_widget_html)
-            if mobj is None:
-                break
-            paging = mobj.group('paging')
-        return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
-
-
-class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
-    IE_NAME = 'youtube:recommended'
-    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
-    _FEED_NAME = 'recommended'
-    _PLAYLIST_TITLE = 'Youtube Recommended videos'
-
-
-class YoutubeWatchLaterIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:watchlater'
-    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
-
-    _TESTS = []  # override PlaylistIE tests
-
-    def _real_extract(self, url):
-        return self._extract_playlist('WL')
-
-
-class YoutubeHistoryIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:history'
-    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
-    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
-    _TESTS = []
-
-    def _real_extract(self, url):
-        title = 'Youtube History'
-        page = self._download_webpage('https://www.youtube.com/feed/history', title)
+        page = self._download_webpage(
+            'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
 
         # The extraction process is the same as for playlists, but the regex
         # for the video ids doesn't contain an index
@@ -1692,17 +1632,25 @@ class YoutubeHistoryIE(YoutubePlaylistIE):
                 break
 
             more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), title,
+                'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
                 'Downloading page #%s' % page_num,
                 transform_source=uppercase_escape)
             content_html = more['content_html']
             more_widget_html = more['load_more_widget_html']
 
-        return {
-            '_type': 'playlist',
-            'title': title,
-            'entries': self._ids_to_results(ids),
-        }
+        return self.playlist_result(
+            self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
+
+
+class YoutubeWatchLaterIE(YoutubePlaylistIE):
+    IE_NAME = 'youtube:watchlater'
+    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
+
+    _TESTS = []  # override PlaylistIE tests
+
+    def _real_extract(self, url):
+        return self._extract_playlist('WL')
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
@@ -1717,42 +1665,25 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
         return self.url_result(playlist_id, 'YoutubePlaylist')
 
 
-class YoutubeSubscriptionsIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:subscriptions'
+class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
+    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
+    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
+    _FEED_NAME = 'recommended'
+    _PLAYLIST_TITLE = 'Youtube Recommended videos'
+
+
+class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
-    _TESTS = []
+    _FEED_NAME = 'subscriptions'
+    _PLAYLIST_TITLE = 'Youtube Subscriptions'
 
-    def _real_extract(self, url):
-        title = 'Youtube Subscriptions'
-        page = self._download_webpage('https://www.youtube.com/feed/subscriptions', title)
 
-        # The extraction process is the same as for playlists, but the regex
-        # for the video ids doesn't contain an index
-        ids = []
-        more_widget_html = content_html = page
-
-        for page_num in itertools.count(1):
-            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
-            new_ids = orderedSet(matches)
-            ids.extend(new_ids)
-
-            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-            if not mobj:
-                break
-
-            more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), title,
-                'Downloading page #%s' % page_num,
-                transform_source=uppercase_escape)
-            content_html = more['content_html']
-            more_widget_html = more['load_more_widget_html']
-
-        return {
-            '_type': 'playlist',
-            'title': title,
-            'entries': self._ids_to_results(ids),
-        }
+class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
+    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
+    _FEED_NAME = 'history'
+    _PLAYLIST_TITLE = 'Youtube History'
 
 
 class YoutubeTruncatedURLIE(InfoExtractor):

From 62c95fd5fcb8dbea2faeb4edac4c5177cbac5912 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 21:42:34 +0600
Subject: [PATCH 0118/2145] [youtube:feed] Check each 'load more' portion for
 unique video ids

---
 youtube_dl/extractor/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9096a2975..1f9940cf5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1621,10 +1621,16 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         # for the video ids doesn't contain an index
         ids = []
         more_widget_html = content_html = page
-
         for page_num in itertools.count(1):
             matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
-            new_ids = orderedSet(matches)
+
+            # 'recommended' feed has infinite 'load more' and each new portion spins
+            # the same videos in (sometimes) slightly different order, so we'll check
+            # for unicity and break when portion has no new videos
+            new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
+            if not new_ids:
+                break
+
             ids.extend(new_ids)
 
             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)

From e9ca615a9872e85a6986061fdf54257244ce1f77 Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 19:57:54 +0400
Subject: [PATCH 0119/2145] New test

---
 youtube_dl/extractor/sportbox.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 695b3ff82..cb1515eff 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -30,10 +30,29 @@ class SportBoxIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
-        }, {
+        }, 
+        {
+            'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',            
+            'md5': 'ff56a598c2cf411a9a38a69709e97079',
+            'info_dict': {
+                'id': '211355',
+                'ext': 'mp4',
+                'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+                'description': '16 детских коллективов приняли участие в суперфинале турнира «Поле славы боевой».',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1426237001,
+                'upload_date': '20150313',
+                'duration': 292,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        }, 
+        {
             'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
             'only_matching': True,
-        }
+        },
     ]
 
     def _real_extract(self, url):

From 34fe5a94baf9e7ea437de68621a5fa73780c0f17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 18:42:59 +0200
Subject: [PATCH 0120/2145] [gamespot] Add support for videos that don't use
 'f4m_stream' (fixes #5707)

---
 youtube_dl/extractor/gamespot.py | 60 +++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 47373e215..5927455f6 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class GameSpotIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
         'info_dict': {
@@ -23,8 +23,16 @@ class GameSpotIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Arma 3 - Community Guide: SITREP I',
             'description': 'Check out this video where some of the basics of Arma 3 is explained.',
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.gamespot.com/videos/the-witcher-3-wild-hunt-xbox-one-now-playing/2300-6424837/',
+        'info_dict': {
+            'id': 'gs-2300-6424837',
+            'ext': 'flv',
+            'title': 'The Witcher 3: Wild Hunt [Xbox ONE]  - Now Playing',
+            'description': 'Join us as we take a look at the early hours of The Witcher 3: Wild Hunt and more.',
+        },
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
@@ -32,25 +40,37 @@ class GameSpotIE(InfoExtractor):
         data_video_json = self._search_regex(
             r'data-video=["\'](.*?)["\']', webpage, 'data video')
         data_video = json.loads(unescapeHTML(data_video_json))
+        streams = data_video['videoStreams']
 
-        # Transform the manifest url to a link to the mp4 files
-        # they are used in mobile devices.
-        f4m_url = data_video['videoStreams']['f4m_stream']
-        f4m_path = compat_urlparse.urlparse(f4m_url).path
-        QUALITIES_RE = r'((,\d+)+,?)'
-        qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
-        http_path = f4m_path[1:].split('/', 1)[1]
-        http_template = re.sub(QUALITIES_RE, r'%s', http_path)
-        http_template = http_template.replace('.csmil/manifest.f4m', '')
-        http_template = compat_urlparse.urljoin(
-            'http://video.gamespotcdn.com/', http_template)
         formats = []
-        for q in qualities:
-            formats.append({
-                'url': http_template % q,
-                'ext': 'mp4',
-                'format_id': q,
-            })
+        f4m_url = streams.get('f4m_stream')
+        if f4m_url is not None:
+            # Transform the manifest url to a link to the mp4 files
+            # they are used in mobile devices.
+            f4m_path = compat_urlparse.urlparse(f4m_url).path
+            QUALITIES_RE = r'((,\d+)+,?)'
+            qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
+            http_path = f4m_path[1:].split('/', 1)[1]
+            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
+            http_template = http_template.replace('.csmil/manifest.f4m', '')
+            http_template = compat_urlparse.urljoin(
+                'http://video.gamespotcdn.com/', http_template)
+            for q in qualities:
+                formats.append({
+                    'url': http_template % q,
+                    'ext': 'mp4',
+                    'format_id': q,
+                })
+        else:
+            for quality in ['sd', 'hd']:
+                # It's actually a link to a flv file
+                flv_url = streams.get('f4m_{0}'.format(quality))
+                if flv_url is not None:
+                    formats.append({
+                        'url': flv_url,
+                        'ext': 'flv',
+                        'format_id': quality,
+                    })
 
         return {
             'id': data_video['guid'],

From eeb23eb7ea6953d7e90ccf669cd0e636d10b2b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 18:44:08 +0200
Subject: [PATCH 0121/2145] [gamespot] The protocol is not optional

---
 youtube_dl/extractor/gamespot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 5927455f6..2d33fa7f5 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'http://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',

From 3a7382950b6f498f50173c8813f6cb1db3739277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 22:50:44 +0600
Subject: [PATCH 0122/2145] [sportbox:embed] Add extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/sportbox.py | 138 +++++++++++++++++--------------
 2 files changed, 82 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8ec0c1032..f293bc2a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -502,7 +502,10 @@ from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
 from .sport5 import Sport5IE
-from .sportbox import SportBoxIE
+from .sportbox import (
+    SportBoxIE,
+    SportBoxEmbedIE,
+)
 from .sportdeutschland import SportDeutschlandIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index cb1515eff..10c45eb74 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
     parse_iso8601,
@@ -12,48 +13,30 @@ from ..utils import (
 
 class SportBoxIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
-    _TESTS = [
-        {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',            
-            'md5': 'ff56a598c2cf411a9a38a69709e97079',
-            'info_dict': {
-                'id': '80822',
-                'ext': 'mp4',
-                'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-                'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'timestamp': 1411896237,
-                'upload_date': '20140928',
-                'duration': 4846,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }, 
-        {
-            'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',            
-            'md5': 'ff56a598c2cf411a9a38a69709e97079',
-            'info_dict': {
-                'id': '211355',
-                'ext': 'mp4',
-                'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
-                'description': '16 детских коллективов приняли участие в суперфинале турнира «Поле славы боевой».',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'timestamp': 1426237001,
-                'upload_date': '20150313',
-                'duration': 292,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }, 
-        {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
-            'only_matching': True,
+    _TESTS = [{
+        'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+        'md5': 'ff56a598c2cf411a9a38a69709e97079',
+        'info_dict': {
+            'id': '80822',
+            'ext': 'mp4',
+            'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
+            'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1411896237,
+            'upload_date': '20140928',
+            'duration': 4846,
         },
-    ]
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
+        'only_matching': True,
+    }, {
+        'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -61,39 +44,74 @@ class SportBoxIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        sobj = re.search(r'src="/vdl/player/(?P<media_type>\w+)/(?P<video_id>\d+)"', webpage)
-        if (sobj):
-            video_id = sobj.group('video_id')
-            media_type = sobj.group('media_type')
-        else:
-            raise RegexNotFoundError('Unable to extract video_id')
-
-        player = self._download_webpage(
-            'http://news.sportbox.ru/vdl/player/%s/%s' % (media_type, video_id),
-            display_id, 'Downloading player webpage')
-
-        hls = self._search_regex(
-            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]+([^\"]+)['\"]+", player, 'hls file')
-
-        formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
+        player = self._search_regex(
+            r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
 
         title = self._html_search_regex(
             r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r'(?s)<div itemprop="description">(.+?)</div>', webpage, 'description', fatal=False)
+            r'(?s)<div itemprop="description">(.+?)</div>',
+            webpage, 'description', fatal=False)
         thumbnail = self._og_search_thumbnail(webpage)
         timestamp = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
+            r'<span itemprop="uploadDate">([^<]+)</span>',
+            webpage, 'timestamp', fatal=False))
         duration = parse_duration(self._html_search_regex(
-            r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
+            r'<meta itemprop="duration" content="PT([^"]+)">',
+            webpage, 'duration', fatal=False))
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'url': compat_urlparse.urljoin(url, '/%s' % player),
             'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
             'duration': duration,
+        }
+
+
+class SportBoxEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://news\.sportbox\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://news.sportbox.ru/vdl/player/ci/211355',
+        'info_dict': {
+            'id': '211355',
+            'ext': 'mp4',
+            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://news.sportbox.ru/vdl/player?nid=370908&only_player=1&autostart=false&playeri=2&height=340&width=580',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        hls = self._search_regex(
+            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]([^'\"]+)['\"]",
+            webpage, 'hls file')
+
+        formats = self._extract_m3u8_formats(hls, video_id, 'mp4')
+
+        title = self._search_regex(
+            r'sportboxPlayer\.node_title\s*=\s*"([^"]+)"', webpage, 'title')
+
+        thumbnail = self._search_regex(
+            r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
         }

From e8cfacae3710c2c225488e4b2d41b84268217a55 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 15 May 2015 17:57:32 +0100
Subject: [PATCH 0123/2145] [CBSNewsIE] Relax thumbnail regex so test passes

---
 youtube_dl/extractor/cbsnews.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7e47960ab..52e61d85b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -32,7 +32,7 @@ class CBSNewsIE(InfoExtractor):
                 'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
                 'ext': 'flv',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
-                'thumbnail': 'http://cbsnews2.cbsistatic.com/hub/i/r/2014/04/04/0c9fbc66-576b-41ca-8069-02d122060dd2/thumbnail/140x90/6dad7a502f88875ceac38202984b6d58/en-0404-werner-replace-640x360.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
             },
             'params': {

From 1436a6835e0f3489a4c37cca3da5087567b68158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:08:44 +0600
Subject: [PATCH 0124/2145] [sportbox:embed] Add `_extract_urls`

---
 youtube_dl/extractor/sportbox.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 10c45eb74..a869a1b25 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -91,6 +91,12 @@ class SportBoxEmbedIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src="(https?://news\.sportbox\.ru/vdl/player[^"]+)"',
+            webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From ef28a6cb26630f8f198a72eee34a2b5c8bd2f802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:09:10 +0600
Subject: [PATCH 0125/2145] [sportbox:embed] Relax thumbnail

---
 youtube_dl/extractor/sportbox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index a869a1b25..8686f9d11 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -113,7 +113,7 @@ class SportBoxEmbedIE(InfoExtractor):
 
         thumbnail = self._search_regex(
             r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
+            webpage, 'thumbnail', default=None)
 
         return {
             'id': video_id,

From d40a3b5b55973d7ed65538179b71990c1828845a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:09:34 +0600
Subject: [PATCH 0126/2145] [generic] Add support for sportbox embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3d756e848..9230c3bb0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,6 +32,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
+from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
@@ -1229,6 +1230,11 @@ class GenericIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
+        # Look for embedded SportBox player
+        sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
+        if sportbox_urls:
+            return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
+
         # Look for embedded TED player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)

From b827a6015c145d67a4d4e9ea38aa54ebe347d3fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:18:21 +0600
Subject: [PATCH 0127/2145] [generic] Add test for sportbox embeds

---
 youtube_dl/extractor/generic.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9230c3bb0..610e33091 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -225,6 +225,37 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # SportBox embed
+        {
+            'url': 'http://www.vestifinance.ru/articles/25753',
+            'info_dict': {
+                'id': '25753',
+                'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '370908',
+                    'title': 'Госзаказ. День 3',
+                    'ext': 'mp4',
+                }
+            }, {
+                'info_dict': {
+                    'id': '370905',
+                    'title': 'Госзаказ. День 2',
+                    'ext': 'mp4',
+                }
+            }, {
+                'info_dict': {
+                    'id': '370902',
+                    'title': 'Госзаказ. День 1',
+                    'ext': 'mp4',
+                }
+            }],
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        },
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',

From 25c3a7348f7971c0af32dcea2d7fd57bd5c63f05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:23:51 +0600
Subject: [PATCH 0128/2145] [generic] Fix typo

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 610e33091..9a7b0d25d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1425,7 +1425,7 @@ class GenericIE(InfoExtractor):
         # Look for Senate ISVP iframe
         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
         if senate_isvp_url:
-            return self.url_result(surl, 'SenateISVP')
+            return self.url_result(senate_isvp_url, 'SenateISVP')
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):

From 70d0d43b5eeff04b41b089e499401e38c115e456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:32:25 +0600
Subject: [PATCH 0129/2145] [rts] Check formats (Closes #5711)

---
 youtube_dl/extractor/rts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index d0981115d..9fbe239d8 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -190,6 +190,7 @@ class RTSIE(InfoExtractor):
                 'tbr': media['rate'] or extract_bitrate(media['url']),
             } for media in info['media'] if media.get('rate')])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         return {

From 0d7f03642976e7859e290b06db41d20a4bfd3a38 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 May 2015 15:43:13 +0800
Subject: [PATCH 0130/2145] [viki] Add support for shows

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/viki.py     | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f293bc2a4..cb6635610 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -639,7 +639,10 @@ from .vine import (
     VineIE,
     VineUserIE,
 )
-from .viki import VikiIE
+from .viki import (
+    VikiIE,
+    VikiShowIE,
+)
 from .vk import (
     VKIE,
     VKUserVideosIE,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index cf6af1e5c..4d185c0e6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -145,3 +145,36 @@ class VikiIE(InfoExtractor):
                 'ext': 'vtt',
             }]
         return res
+
+
+class VikiShowIE(InfoExtractor):
+    IE_NAME = 'viki:show'
+    _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _TESTS = [{
+        'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
+        'info_dict': {
+            'id': '50c',
+            'title': 'Boys Over Flowers',
+            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
+        },
+        'playlist_count': 25,
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        show_page = self._download_webpage(url, show_id, 'Download show page')
+
+        title = self._og_search_title(show_page)
+        description = self._og_search_description(show_page)
+
+        show_json = self._download_json(
+            'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
+            show_id, note='Retrieve show json', errnote='Unable to get show json'
+        )
+        entries = []
+        for video in show_json['response']:
+            video_id = video['id']
+            entries.append(self.url_result(
+                'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+
+        return self.playlist_result(entries, show_id, title, description)

From 2f3bdab2b90c6695c0a478f352967b0c9da4f23f Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 May 2015 15:56:37 +0800
Subject: [PATCH 0131/2145] [viki] Fix code format

---
 youtube_dl/extractor/viki.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 4d185c0e6..40a73f561 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -169,8 +169,7 @@ class VikiShowIE(InfoExtractor):
 
         show_json = self._download_json(
             'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
-            show_id, note='Retrieve show json', errnote='Unable to get show json'
-        )
+            show_id, note='Retrieve show json', errnote='Unable to get show json')
         entries = []
         for video in show_json['response']:
             video_id = video['id']

From 1c97b0a777f52c520587e93e7e61721fa6195977 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 20:00:40 +0600
Subject: [PATCH 0132/2145] [ooyala:external] Add extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/ooyala.py   | 221 +++++++++++++++++++------------
 2 files changed, 137 insertions(+), 89 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f293bc2a4..1731f4fb2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -376,7 +376,10 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
-from .ooyala import OoyalaIE
+from .ooyala import (
+    OoyalaIE,
+    OoyalaExternalIE,
+)
 from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index c0e6d643d..9f4fe5b29 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -12,7 +12,100 @@ from ..utils import (
 )
 
 
-class OoyalaIE(InfoExtractor):
+class OoyalaBaseIE(InfoExtractor):
+
+    def _extract_result(self, info, more_info):
+        embedCode = info['embedCode']
+        video_url = info.get('ipad_url') or info['url']
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': 'mp4',
+            }]
+
+        return {
+            'id': embedCode,
+            'title': unescapeHTML(info['title']),
+            'formats': formats,
+            'description': unescapeHTML(more_info['description']),
+            'thumbnail': more_info['promo'],
+        }
+
+    def _extract(self, player_url, video_id):
+        player = self._download_webpage(player_url, video_id)
+        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
+                                        player, 'mobile player url')
+        # Looks like some videos are only available for particular devices
+        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
+        # is only available for ipad)
+        # Working around with fetching URLs for all the devices found starting with 'unknown'
+        # until we succeed or eventually fail for each device.
+        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
+        devices.remove('unknown')
+        devices.insert(0, 'unknown')
+        for device in devices:
+            mobile_player = self._download_webpage(
+                '%s&device=%s' % (mobile_url, device), video_id,
+                'Downloading mobile player JS for %s device' % device)
+            videos_info = self._search_regex(
+                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
+                mobile_player, 'info', fatal=False, default=None)
+            if videos_info:
+                break
+
+        if not videos_info:
+            formats = []
+            auth_data = self._download_json(
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                video_id)
+
+            cur_auth_data = auth_data['authorization_data'][video_id]
+
+            for stream in cur_auth_data['streams']:
+                formats.append({
+                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
+                    'ext': stream.get('delivery_type'),
+                    'format': stream.get('video_codec'),
+                    'format_id': stream.get('profile'),
+                    'width': int_or_none(stream.get('width')),
+                    'height': int_or_none(stream.get('height')),
+                    'abr': int_or_none(stream.get('audio_bitrate')),
+                    'vbr': int_or_none(stream.get('video_bitrate')),
+                })
+            if formats:
+                return {
+                    'id': video_id,
+                    'formats': formats,
+                    'title': 'Ooyala video',
+                }
+
+            if not cur_auth_data['authorized']:
+                raise ExtractorError(cur_auth_data['message'], expected=True)
+
+        if not videos_info:
+            raise ExtractorError('Unable to extract info')
+        videos_info = videos_info.replace('\\"', '"')
+        videos_more_info = self._search_regex(
+            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
+        videos_info = json.loads(videos_info)
+        videos_more_info = json.loads(videos_more_info)
+
+        if videos_more_info.get('lineup'):
+            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': unescapeHTML(videos_more_info['title']),
+                'entries': videos,
+            }
+        else:
+            return self._extract_result(videos_info[0], videos_more_info)
+
+
+class OoyalaIE(OoyalaBaseIE):
     _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'
 
     _TESTS = [
@@ -57,95 +150,47 @@ class OoyalaIE(InfoExtractor):
         return cls.url_result(cls._url_for_embed_code(embed_code),
                               ie=cls.ie_key())
 
-    def _extract_result(self, info, more_info):
-        embedCode = info['embedCode']
-        video_url = info.get('ipad_url') or info['url']
+    def _real_extract(self, url):
+        embed_code = self._match_id(url)
+        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
+        return self._extract(player_url, embed_code)
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': 'mp4',
-            }]
 
-        return {
-            'id': embedCode,
-            'title': unescapeHTML(info['title']),
-            'formats': formats,
-            'description': unescapeHTML(more_info['description']),
-            'thumbnail': more_info['promo'],
-        }
+class OoyalaExternalIE(OoyalaBaseIE):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        ooyalaexternal:|
+                        https?://.+?\.ooyala\.com/.*?\bexternalId=
+                    )
+                    (?P<partner_id>[^:]+)
+                    :
+                    (?P<id>.+)
+                    (?:
+                        :|
+                        .*?&pcode=
+                    )
+                    (?P<pcode>.+?)
+                    (&|$)
+                    '''
+
+    _TEST = {
+        'url': 'https://player.ooyala.com/player.js?externalId=espn:10365079&pcode=1kNG061cgaoolOncv54OAO1ceO-I&adSetCode=91cDU6NuXTGKz3OdjOxFdAgJVtQcKJnI&callback=handleEvents&hasModuleParams=1&height=968&playerBrandingId=7af3bd04449c444c964f347f11873075&targetReplaceId=videoPlayer&width=1656&wmode=opaque&allowScriptAccess=always',
+        'info_dict': {
+            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
+            'ext': 'mp4',
+            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'description': '',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        embedCode = mobj.group('id')
-        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embedCode
-        player = self._download_webpage(player_url, embedCode)
-        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
-                                        player, 'mobile player url')
-        # Looks like some videos are only available for particular devices
-        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
-        # is only available for ipad)
-        # Working around with fetching URLs for all the devices found starting with 'unknown'
-        # until we succeed or eventually fail for each device.
-        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
-        devices.remove('unknown')
-        devices.insert(0, 'unknown')
-        for device in devices:
-            mobile_player = self._download_webpage(
-                '%s&device=%s' % (mobile_url, device), embedCode,
-                'Downloading mobile player JS for %s device' % device)
-            videos_info = self._search_regex(
-                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
-                mobile_player, 'info', fatal=False, default=None)
-            if videos_info:
-                break
-
-        if not videos_info:
-            formats = []
-            auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
-                embedCode)
-
-            cur_auth_data = auth_data['authorization_data'][embedCode]
-
-            for stream in cur_auth_data['streams']:
-                formats.append({
-                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
-                    'ext': stream.get('delivery_type'),
-                    'format': stream.get('video_codec'),
-                    'format_id': stream.get('profile'),
-                    'width': int_or_none(stream.get('width')),
-                    'height': int_or_none(stream.get('height')),
-                    'abr': int_or_none(stream.get('audio_bitrate')),
-                    'vbr': int_or_none(stream.get('video_bitrate')),
-                })
-            if formats:
-                return {
-                    'id': embedCode,
-                    'formats': formats,
-                    'title': 'Ooyala video',
-                }
-
-            if not cur_auth_data['authorized']:
-                raise ExtractorError(cur_auth_data['message'], expected=True)
-
-        if not videos_info:
-            raise ExtractorError('Unable to extract info')
-        videos_info = videos_info.replace('\\"', '"')
-        videos_more_info = self._search_regex(
-            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
-        videos_info = json.loads(videos_info)
-        videos_more_info = json.loads(videos_more_info)
-
-        if videos_more_info.get('lineup'):
-            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
-            return {
-                '_type': 'playlist',
-                'id': embedCode,
-                'title': unescapeHTML(videos_more_info['title']),
-                'entries': videos,
-            }
-        else:
-            return self._extract_result(videos_info[0], videos_more_info)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+        pcode = mobj.group('pcode')
+        player_url = 'http://player.ooyala.com/player.js?externalId=%s:%s&pcode=%s' % (partner_id, video_id, pcode)
+        return self._extract(player_url, video_id)

From 9354a5fad4521687eb9f08c1a42848621857400b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 20:15:31 +0600
Subject: [PATCH 0133/2145] [ooyala] Fix unresolved reference

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 9f4fe5b29..a262a9f6d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -59,7 +59,7 @@ class OoyalaBaseIE(InfoExtractor):
         if not videos_info:
             formats = []
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (video_id, video_id),
                 video_id)
 
             cur_auth_data = auth_data['authorization_data'][video_id]

From ef2dcbe4adce4478d409397faaae7ec6453ecf7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 21:07:29 +0600
Subject: [PATCH 0134/2145] [sbs] Fix extraction (Closes #5725)

---
 youtube_dl/extractor/sbs.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index b8775c2f9..3073e5e86 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -33,16 +33,18 @@ class SBSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        release_urls_json = js_to_json(self._search_regex(
+        player = self._search_regex(
             r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
-            webpage, ''))
-        release_urls = json.loads(release_urls_json)
-        theplatform_url = (
-            release_urls.get('progressive') or release_urls.get('standard'))
+            webpage, 'player')
+        player = re.sub(r"'\s*\+\s*[\da-zA-Z_]+\s*\+\s*'", '', player)
+
+        release_urls = self._parse_json(js_to_json(player), video_id)
+
+        theplatform_url = release_urls.get('progressive') or release_urls['standard']
 
         title = remove_end(self._og_search_title(webpage), ' (The Feed)')
         description = self._html_search_meta('description', webpage)
@@ -52,7 +54,6 @@ class SBSIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': theplatform_url,
-
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 7e760fc18897663db7c0717434e28a8cca9f3810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 21:14:19 +0600
Subject: [PATCH 0135/2145] [espn] Add extractor (#4396)

Unfinished
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/espn.py     | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/espn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1731f4fb2..6b19eb6f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -141,6 +141,7 @@ from .engadget import EngadgetIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
+from .espn import ESPNIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
new file mode 100644
index 000000000..e6f8f0337
--- /dev/null
+++ b/youtube_dl/extractor/espn.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ESPNIE(InfoExtractor):
+    _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
+    _WORKING = False
+    _TESTS = [{
+        'url': 'http://espn.go.com/video/clip?id=10365079',
+        'info_dict': {
+            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
+            'ext': 'mp4',
+            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'description': '',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/nba/recap?gameId=400793786',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'class="video-play-button"[^>]+data-id="(\d+)',
+            webpage, 'video id')
+
+        player = self._download_webpage(
+            'https://espn.go.com/video/iframe/twitter/?id=%s' % video_id, video_id)
+
+        pcode = self._search_regex(
+            r'["\']pcode=([^"\']+)["\']', player, 'pcode')
+
+        return self.url_result(
+            'ooyalaexternal:espn:%s:%s' % (video_id, pcode),
+            'OoyalaExternal')

From 4d52f2eb7f4b16ea5491f20abf0b29a1fcb24a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 16 May 2015 18:38:28 +0200
Subject: [PATCH 0136/2145] [sbs] Remove unused import

---
 youtube_dl/extractor/sbs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 3073e5e86..d4bd1a0d7 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import json
 import re
 from .common import InfoExtractor
 from ..utils import (

From 1c18de00192d195357989861563cc1fad9256128 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 May 2015 01:38:50 +0800
Subject: [PATCH 0137/2145] [viki] Add proper paging and include clips

---
 youtube_dl/extractor/viki.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 40a73f561..4d477b03c 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -93,7 +93,7 @@ class VikiIE(InfoExtractor):
                     'Video %s is blocked from your location.' % video_id,
                     expected=True)
             else:
-                raise ExtractorError('Viki said: ' + err_msg)
+                raise ExtractorError('Viki said: %s %s' % (err_msg, url))
         mobj = re.search(
             r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
         if not mobj:
@@ -157,7 +157,15 @@ class VikiShowIE(InfoExtractor):
             'title': 'Boys Over Flowers',
             'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
         },
-        'playlist_count': 25,
+        'playlist_count': 70,
+    }, {
+        'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
+        'info_dict': {
+            'id': '1354c',
+            'title': 'Poor Nastya [COMPLETE]',
+            'description': 'md5:05bf5471385aa8b21c18ad450e350525',
+        },
+        'playlist_count': 127,
     }]
 
     def _real_extract(self, url):
@@ -167,13 +175,16 @@ class VikiShowIE(InfoExtractor):
         title = self._og_search_title(show_page)
         description = self._og_search_description(show_page)
 
-        show_json = self._download_json(
-            'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
-            show_id, note='Retrieve show json', errnote='Unable to get show json')
         entries = []
-        for video in show_json['response']:
-            video_id = video['id']
-            entries.append(self.url_result(
-                'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+        for video_type in ['episodes', 'clips']:
+            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=25&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type)
+            while json_url is not None:
+                show_json = self._download_json(
+                    json_url, show_id, note='Retrieve show json', errnote='Unable to get show json')
+                for video in show_json['response']:
+                    video_id = video['id']
+                    entries.append(self.url_result(
+                        'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+                json_url = show_json['pagination']['next']
 
         return self.playlist_result(entries, show_id, title, description)

From baa43cbaf01a575eacb8e1bb39c7200f68c36daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 02:59:35 +0600
Subject: [PATCH 0138/2145] [extractor/common] Relax valid url check verbosity

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 65bb77086..cecf917ff 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -786,8 +786,8 @@ class InfoExtractor(object):
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
-                self.report_warning(
-                    '%s URL is invalid, skipping' % item, video_id)
+                self.to_screen(
+                    '%s: %s URL is invalid, skipping' % (video_id, item))
                 return False
             raise
 

From bc0f937b55aae6ce731d259a7658b0281c2e62ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 03:01:52 +0600
Subject: [PATCH 0139/2145] [tv2] Add extractor (#5724)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tv2.py      | 93 ++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 youtube_dl/extractor/tv2.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6b19eb6f8..fb4f63ca3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -572,6 +572,7 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
+from .tv2 import TV2IE
 from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
new file mode 100644
index 000000000..2dcc0e971
--- /dev/null
+++ b/youtube_dl/extractor/tv2.py
@@ -0,0 +1,93 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
+)
+
+
+class TV2IE(InfoExtractor):
+    _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tv2.no/v/916509/',
+        'md5': '9cb9e3410b18b515d71892f27856e9b1',
+        'info_dict': {
+            'id': '916509',
+            'ext': 'flv',
+            'title': 'Se Gryttens hyllest av Steven Gerrard',
+            'description': 'TV 2 Sportens huspoet tar avskjed med Liverpools kaptein Steven Gerrard.',
+            'timestamp': 1431715610,
+            'upload_date': '20150515',
+            'duration': 156.967,
+            'view_count': int,
+            'categories': list,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        formats = []
+        format_urls = []
+        for protocol in ('HDS', 'HLS'):
+            data = self._download_json(
+                'http://sumo.tv2.no/api/web/asset/%s/play.json?protocol=%s&videoFormat=SMIL+ISMUSP' % (video_id, protocol),
+                video_id, 'Downloading play JSON')['playback']
+            for item in data['items']['item']:
+                video_url = item.get('url')
+                if not video_url or video_url in format_urls:
+                    continue
+                format_id = '%s-%s' % (protocol.lower(), item.get('mediaFormat'))
+                if not self._is_valid_url(video_url, video_id, format_id):
+                    continue
+                format_urls.append(video_url)
+                ext = determine_ext(video_url)
+                if ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        video_url, video_id, f4m_id=format_id))
+                elif ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=format_id))
+                elif ext == 'ism' or video_url.endswith('.ism/Manifest'):
+                    pass
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                        'tbr': int_or_none(item.get('bitrate')),
+                        'filesize': int_or_none(item.get('fileSize')),
+                    })
+        self._sort_formats(formats)
+
+        asset = self._download_json(
+            'http://sumo.tv2.no/api/web/asset/%s.json' % video_id,
+            video_id, 'Downloading metadata JSON')['asset']
+
+        title = asset['title']
+        description = asset.get('description')
+        timestamp = parse_iso8601(asset.get('createTime'))
+        duration = float_or_none(asset.get('accurateDuration') or asset.get('duration'))
+        view_count = int_or_none(asset.get('views'))
+        categories = asset.get('keywords', '').split(',')
+
+        thumbnails = [{
+            'id': thumbnail.get('@type'),
+            'url': thumbnail.get('url'),
+        } for _, thumbnail in asset.get('imageVersions', {}).items()]
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'categories': categories,
+            'formats': formats,
+        }

From 588b82bbf8c90981c54f180eca40e6c743f8f89f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 03:32:53 +0600
Subject: [PATCH 0140/2145] [tv2:article] Add extractor (Closes #5724)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/tv2.py      | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb4f63ca3..6f8c261d5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -572,7 +572,10 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
-from .tv2 import TV2IE
+from .tv2 import (
+    TV2IE,
+    TV2ArticleIE,
+)
 from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 2dcc0e971..fa338b936 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -1,12 +1,15 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
     float_or_none,
     parse_iso8601,
+    remove_end,
 )
 
 
@@ -91,3 +94,33 @@ class TV2IE(InfoExtractor):
             'categories': categories,
             'formats': formats,
         }
+
+
+class TV2ArticleIE(InfoExtractor):
+    _VALID_URL = 'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
+        'info_dict': {
+            'id': '6930542',
+            'title': 'Russen hetses etter pingvintyveri – innrømmer å ha åpnet luken på buret',
+            'description': 'md5:339573779d3eea3542ffe12006190954',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.tv2.no/a/6930542',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.tv2.no/v/%s' % video_id, 'TV2')
+            for video_id in re.findall(r'data-assetid="(\d+)"', webpage)]
+
+        title = remove_end(self._og_search_title(webpage), ' - TV2.no')
+        description = remove_end(self._og_search_description(webpage), ' - TV2.no')
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 8da0e0e94682faa0463f33d991df70a2402b5a86 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 May 2015 06:19:38 +0800
Subject: [PATCH 0141/2145] [viki] Change IE name to channel, better message
 output

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/viki.py     | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cb6635610..21f7b7290 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -641,7 +641,7 @@ from .vine import (
 )
 from .viki import (
     VikiIE,
-    VikiShowIE,
+    VikiChannelIE,
 )
 from .vk import (
     VKIE,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 4d477b03c..9bdbdc3e4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -147,8 +147,8 @@ class VikiIE(InfoExtractor):
         return res
 
 
-class VikiShowIE(InfoExtractor):
-    IE_NAME = 'viki:show'
+class VikiChannelIE(InfoExtractor):
+    IE_NAME = 'viki:channel'
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
@@ -167,6 +167,7 @@ class VikiShowIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
+    _PER_PAGE = 25
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
@@ -177,10 +178,12 @@ class VikiShowIE(InfoExtractor):
 
         entries = []
         for video_type in ['episodes', 'clips']:
-            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=25&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type)
+            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type, self._PER_PAGE)
             while json_url is not None:
                 show_json = self._download_json(
-                    json_url, show_id, note='Retrieve show json', errnote='Unable to get show json')
+                    json_url, show_id,
+                    note='Downloading %s json page #%s' %
+                         (video_type, re.search(r'[?&]page=([0-9]+)', json_url).group(1)))
                 for video in show_json['response']:
                     video_id = video['id']
                     entries.append(self.url_result(

From 725652e9247e1171110b624d748e20fa1c88260e Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sat, 16 May 2015 19:50:58 -0500
Subject: [PATCH 0142/2145] [karrierevideos] add support for
 www.karrierevideos.at (closes #5354)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/karrierevideos.py | 52 ++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/karrierevideos.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb4f63ca3..d131d3ec3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -244,6 +244,7 @@ from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
+from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
new file mode 100644
index 000000000..59d29e845
--- /dev/null
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KarriereVideosIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P<id>[a-z-]+)'
+    _TEST = {
+        'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
+        'info_dict': {
+            'id': 'altenpflegerin',
+            'ext': 'mp4',
+            'title': 'AltenpflegerIn',
+            'thumbnail': 're:^http://.*\.png\?v=[0-9]+',
+            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2'
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        description = self._html_search_regex(
+            r'<div class="leadtext">\n{0,}?\s{0,}<p>(.*?)</p>',
+            webpage, 'description')
+
+        playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
+        playlist = self._download_xml(
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist,
+            video_id)
+
+        namespace = {
+            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
+        }
+
+        item = playlist.find('tracklist/item')
+        streamer = item.find('jwplayer:streamer', namespace).text
+
+        return {
+            'id': video_id,
+            'title': self._html_search_meta('title', webpage),
+            'description': description,
+            'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
+            'protocol': 'rtmp',
+            'url': streamer.replace('rtmpt', 'http'),
+            'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text,
+            'tc_url': streamer,
+            'ext': 'mp4'
+        }

From ba9d16291b8ace3bd412bcfc0c128c047545e509 Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 17 May 2015 03:35:08 -0500
Subject: [PATCH 0143/2145] manually specify namespace

---
 youtube_dl/extractor/karrierevideos.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index 59d29e845..a05e8ab76 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -29,15 +29,13 @@ class KarriereVideosIE(InfoExtractor):
 
         playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
         playlist = self._download_xml(
-            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist,
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist,
             video_id)
 
-        namespace = {
-            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
-        }
+        namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
 
         item = playlist.find('tracklist/item')
-        streamer = item.find('jwplayer:streamer', namespace).text
+        streamer = item.find('{%s}streamer' % namespace).text
 
         return {
             'id': video_id,
@@ -46,7 +44,7 @@ class KarriereVideosIE(InfoExtractor):
             'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
             'protocol': 'rtmp',
             'url': streamer.replace('rtmpt', 'http'),
-            'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text,
+            'play_path': 'mp4:' + item.find('{%s}file' % namespace).text,
             'tc_url': streamer,
             'ext': 'mp4'
         }

From 4a5a898a8fa392d02102672f9767f33a39a73066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 20:56:03 +0600
Subject: [PATCH 0144/2145] [YoutubeDL] Clarify incompatible formats merge
 message

When `-f` is not specified it's misleading to see `You have requested ...` as user did not actually request any formats.
---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5df889945..58b34e087 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1368,7 +1368,7 @@ class YoutubeDL(object):
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
-                                            ' The formats won\'t be merged')
+                                            ' The formats won\'t be merged.')
                     else:
                         postprocessors = [merger]
 
@@ -1395,8 +1395,8 @@ class YoutubeDL(object):
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         info_dict['ext'] = 'mkv'
-                        self.report_warning('You have requested formats incompatible for merge. '
-                                            'The formats will be merged into mkv')
+                        self.report_warning(
+                            'Requested formats are incompatible for merge and will be merged into mkv.')
                     # Ensure filename always has a correct extension for successful merge
                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
                     if os.path.exists(encodeFilename(filename)):

From fc6e75dd57f3497b99def659b3d0f173b195b7d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 May 2015 11:21:09 +0200
Subject: [PATCH 0145/2145] [instagram] Only recognize https urls (fixes #5739)

http urls redirect to them.
---
 youtube_dl/extractor/instagram.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 65f6ca103..b10755788 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -7,9 +7,9 @@ from ..utils import int_or_none
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https?://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
     _TEST = {
-        'url': 'http://instagram.com/p/aye83DjauH/?foo=bar#abc',
+        'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
         'info_dict': {
             'id': 'aye83DjauH',
@@ -41,11 +41,11 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'http://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {
-        'url': 'http://instagram.com/porsche',
+        'url': 'https://instagram.com/porsche',
         'info_dict': {
             'id': 'porsche',
             'title': 'porsche',

From 5bdc520cf19f404247ec2be1ffc1e83449fa2375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:23:05 +0600
Subject: [PATCH 0146/2145] [xminus] Fix extraction

---
 youtube_dl/extractor/xminus.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index 8c6241aed..7c9d8af6f 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -43,7 +43,7 @@ class XMinusIE(InfoExtractor):
             r'minus_track\.dur_sec=\'([0-9]*?)\'',
             webpage, 'duration', fatal=False))
         filesize_approx = parse_filesize(self._html_search_regex(
-            r'<div class="filesize[^"]*"></div>\s*([0-9.]+\s*[a-zA-Z][bB])',
+            r'<div id="finfo"[^>]*>\s*↓\s*([0-9.]+\s*[a-zA-Z][bB])',
             webpage, 'approximate filesize', fatal=False))
         tbr = int_or_none(self._html_search_regex(
             r'<div class="quality[^"]*"></div>\s*([0-9]+)\s*kbps',
@@ -58,7 +58,7 @@ class XMinusIE(InfoExtractor):
             description = re.sub(' *\r *', '\n', description)
 
         enc_token = self._html_search_regex(
-            r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
+            r'minus_track\.s?tkn="(.+?)"', webpage, 'enc_token')
         token = ''.join(
             c if pos == 3 else compat_chr(compat_ord(c) - 1)
             for pos, c in enumerate(reversed(enc_token)))

From 4f514c7e88d2ce8ebe9c2478183e8797cfb2a4c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:29:41 +0600
Subject: [PATCH 0147/2145] [wimp] Fix youtube extraction (Closes #5690)

---
 youtube_dl/extractor/wimp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index d6dec25ca..f69d46a28 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -37,7 +37,8 @@ class WimpIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", webpage, 'video URL')
+            [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],
+            webpage, 'video URL')
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {

From 2328f2fe684f9a9025217c6f149e92a403a4c437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:34:20 +0600
Subject: [PATCH 0148/2145] [vulture] Fix extraction

---
 youtube_dl/extractor/vulture.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vulture.py b/youtube_dl/extractor/vulture.py
index 1eb24a3d6..faa167e65 100644
--- a/youtube_dl/extractor/vulture.py
+++ b/youtube_dl/extractor/vulture.py
@@ -44,7 +44,7 @@ class VultureIE(InfoExtractor):
         query_webpage = self._download_webpage(
             query_url, display_id, note='Downloading query page')
         params_json = self._search_regex(
-            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n,\n',
+            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n?,\n',
             query_webpage,
             'player params')
         params = json.loads(params_json)

From 5d8dcb5342c97b05c037c8c4e80002540db261b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:39:15 +0600
Subject: [PATCH 0149/2145] [vuclip] Fix extraction

---
 youtube_dl/extractor/vuclip.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index c3fde53f5..a6d9b5fee 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -49,7 +49,7 @@ class VuClipIE(InfoExtractor):
         links_code = self._search_regex(
             r'''(?xs)
                 (?:
-                    <img\s+src="/im/play.gif".*?>|
+                    <img\s+src="[^"]*/play.gif".*?>|
                     <!--\ player\ end\ -->\s*</div><!--\ thumb\ end-->
                 )
                 (.*?)

From 484c9d2d5b669220c24c865947c3f65049916b56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:43:54 +0600
Subject: [PATCH 0150/2145] [vier] Fix extraction

---
 youtube_dl/extractor/vier.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 619039e51..15377097e 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -38,11 +38,14 @@ class VierIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_id = self._search_regex(
-            r'"nid"\s*:\s*"(\d+)"', webpage, 'video id')
+            [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
+            webpage, 'video id')
         application = self._search_regex(
-            r'"application"\s*:\s*"([^"]+)"', webpage, 'application', default='vier_vod')
+            [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
+            webpage, 'application', default='vier_vod')
         filename = self._search_regex(
-            r'"filename"\s*:\s*"([^"]+)"', webpage, 'filename')
+            [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
+            webpage, 'filename')
 
         playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
         formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')

From 2aa64b89b3ac8f387d4c0c27ce7de64bc0ff68de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 May 2015 17:58:53 +0200
Subject: [PATCH 0151/2145] tox: Pass HOME environment variable

Since version 2.0 it only passes a limited set of variables and we need HOME for the tests
---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 00c6e00e3..cd805fe8a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,6 +4,8 @@ envlist = py26,py27,py33,py34
 deps =
    nose
    coverage
+# We need a valid $HOME for test_compat_expanduser
+passenv = HOME
 defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
     --exclude test_subtitles.py --exclude test_write_annotations.py
     --exclude test_youtube_lists.py

From 1b0427e6c433c0b6db5e210db6e3173e19e702ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 00:45:01 +0800
Subject: [PATCH 0152/2145] [utils] Support TTML without default namespace

In a strict sense such TTML is invalid, but Yahoo uses it.
---
 test/test_utils.py  | 15 +++++++++++++++
 youtube_dl/utils.py |  9 ++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index b40107037..e13e11b59 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -621,6 +621,21 @@ Line
 '''
         self.assertEqual(dfxp2srt(dfxp_data), srt_data)
 
+        dfxp_data_no_default_namespace = '''<?xml version="1.0" encoding="UTF-8"?>
+            <tt xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">The first line</p>
+                </div>
+            </body>
+            </tt>'''
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+The first line
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ed9ed9ed6..507f07383 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1848,9 +1848,9 @@ def dfxp2srt(dfxp_data):
         out = str_or_empty(node.text)
 
         for child in node:
-            if child.tag == _x('ttml:br'):
+            if child.tag in (_x('ttml:br'), 'br'):
                 out += '\n' + str_or_empty(child.tail)
-            elif child.tag == _x('ttml:span'):
+            elif child.tag in (_x('ttml:span'), 'span'):
                 out += str_or_empty(parse_node(child))
             else:
                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
@@ -1859,7 +1859,10 @@ def dfxp2srt(dfxp_data):
 
     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p'))
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
+
+    if not paras:
+        raise ValueError('Invalid dfxp/TTML subtitle')
 
     for para, index in zip(paras, itertools.count(1)):
         begin_time = parse_dfxp_time_expr(para.attrib['begin'])

From ecee5724110847b832a6074c66ca4a63758100f4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 00:50:24 +0800
Subject: [PATCH 0153/2145] [yahoo] Add support for closed captions (closes
 #5714)

---
 youtube_dl/extractor/yahoo.py | 18 ++++++++++++++++++
 youtube_dl/utils.py           |  1 +
 2 files changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index bf4e659ac..f9afbdbab 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -15,6 +15,7 @@ from ..utils import (
     unescapeHTML,
     ExtractorError,
     int_or_none,
+    mimetype2ext,
 )
 
 from .nbc import NBCSportsVPlayerIE
@@ -236,6 +237,22 @@ class YahooIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        closed_captions = self._html_search_regex(
+            r'"closedcaptions":(\[[^\]]+\])', webpage, 'closed captions',
+            default='[]')
+
+        cc_json = self._parse_json(closed_captions, video_id, fatal=False)
+        subtitles = {}
+        if cc_json:
+            for closed_caption in cc_json:
+                lang = closed_caption['lang']
+                if lang not in subtitles:
+                    subtitles[lang] = []
+                subtitles[lang].append({
+                    'url': closed_caption['url'],
+                    'ext': mimetype2ext(closed_caption['content_type']),
+                })
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -244,6 +261,7 @@ class YahooIE(InfoExtractor):
             'description': clean_html(meta['description']),
             'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
             'duration': int_or_none(meta.get('duration')),
+            'subtitles': subtitles,
         }
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 507f07383..52d198fa3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1665,6 +1665,7 @@ def mimetype2ext(mt):
     return {
         'x-ms-wmv': 'wmv',
         'x-mp4-fragmented': 'mp4',
+        'ttml+xml': 'ttml',
     }.get(res, res)
 
 

From b813d8caf1b23821036b77b851e42ba0a0ad35a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 01:01:42 +0800
Subject: [PATCH 0154/2145] [qqmusic] Unescape '\\n' in description (#5705)

---
 youtube_dl/extractor/qqmusic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 13113820b..b540033e2 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -26,7 +26,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
-            'description': 'md5:4348ff1dd24036906baa7b6f973f8d30',
+            'description': 'md5:d327722d0361576fde558f1ac68a7065',
         }
     }]
 
@@ -60,6 +60,8 @@ class QQMusicIE(InfoExtractor):
         lrc_content = self._html_search_regex(
             r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)</div>',
             detail_info_page, 'LRC lyrics', default=None)
+        if lrc_content:
+            lrc_content = lrc_content.replace('\\n', '\n')
 
         guid = self.m_r_get_ruin()
 

From d9d747a06ab3b4c36c6063074ffb42aeb185431f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 May 2015 21:28:41 +0600
Subject: [PATCH 0155/2145] [ultimedia] Fix extraction

---
 youtube_dl/extractor/ultimedia.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 96c809eaf..c4751050e 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     ExtractorError,
     qualities,
@@ -44,9 +45,9 @@ class UltimediaIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        deliver_url = self._search_regex(
-            r'<iframe[^>]+src="(https?://(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
-            webpage, 'deliver URL')
+        deliver_url = self._proto_relative_url(self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
+            webpage, 'deliver URL'), compat_urllib_parse_urlparse(url).scheme + ':')
 
         deliver_page = self._download_webpage(
             deliver_url, video_id, 'Downloading iframe page')
@@ -57,7 +58,8 @@ class UltimediaIE(InfoExtractor):
 
         player = self._parse_json(
             self._search_regex(
-                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),
+                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on",
+                deliver_page, 'player'),
             video_id)
 
         quality = qualities(['flash', 'html5'])

From f670ef1c8ebe0329a68b3a3d5c2b7e07ae5c9425 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 13:51:43 +0800
Subject: [PATCH 0156/2145] [dramafever] Add new extractor for dramafever.com

---
 youtube_dl/extractor/__init__.py   |   4 +
 youtube_dl/extractor/dramafever.py | 131 +++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 youtube_dl/extractor/dramafever.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f8c261d5..ca857a75f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -112,6 +112,10 @@ from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
+from .dramafever import (
+    DramaFeverIE,
+    DramaFeverSeriesIE,
+)
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
new file mode 100644
index 000000000..8fac99cc5
--- /dev/null
+++ b/youtube_dl/extractor/dramafever.py
@@ -0,0 +1,131 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class DramaFeverIE(InfoExtractor):
+    IE_NAME = 'dramafever'
+    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)/'
+    _TESTS = [{
+        'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
+        'info_dict': {
+            'id': '4512.1',
+            'ext': 'flv',
+            'title': 'Cooking with Shin 4512.1',
+            'upload_date': '20140702',
+            'description': 'Served at all special occasions and featured in the hit drama Heirs, Shin cooks Red Bean Rice.',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url).replace("/", ".")
+
+        consumer_secret = self._get_consumer_secret(video_id)
+
+        ep_json = self._download_json(
+            "http://www.dramafever.com/amp/episode/feed.json?guid=%s" % video_id,
+            video_id, note='Downloading episode metadata',
+            errnote="Video may not be available for your location")["channel"]["item"]
+
+        title = ep_json["media-group"]["media-title"]
+        description = ep_json["media-group"]["media-description"]
+        thumbnail = ep_json["media-group"]["media-thumbnail"]["@attributes"]["url"]
+        duration = int(ep_json["media-group"]["media-content"][0]["@attributes"]["duration"])
+        mobj = re.match(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", ep_json["pubDate"])
+        upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) if mobj is not None else None
+
+        formats = []
+        for vid_format in ep_json["media-group"]["media-content"]:
+            src = vid_format["@attributes"]["url"]
+            if '.f4m' in src:
+                formats.extend(self._extract_f4m_formats(src, video_id))
+
+        self._sort_formats(formats)
+        video_subtitles = self.extract_subtitles(video_id, consumer_secret)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': video_subtitles,
+        }
+
+    def _get_consumer_secret(self, video_id):
+        df_js = self._download_webpage(
+            "http://www.dramafever.com/static/126960d/v2/js/plugins/jquery.threadedcomments.js", video_id)
+        return self._search_regex(r"'cs': '([0-9a-zA-Z]+)'", df_js, "cs")
+
+    def _get_episodes(self, series_id, consumer_secret, episode_filter=None):
+        _PAGE_SIZE = 60
+
+        curr_page = 1
+        max_pages = curr_page + 1
+        results = []
+        while max_pages >= curr_page:
+            page_url = "http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d" % \
+                       (consumer_secret, series_id, _PAGE_SIZE, curr_page)
+            series = self._download_json(
+                page_url, series_id, note="Downloading series json page #%d" % curr_page)
+            max_pages = series['num_pages']
+            results.extend([ep for ep in series['value'] if episode_filter is None or episode_filter(ep)])
+            curr_page += 1
+        return results
+
+    def _get_subtitles(self, video_id, consumer_secret):
+
+        def match_episode(ep):
+            return ep['guid'] == video_id
+
+        res = None
+        info = self._get_episodes(
+            video_id.split(".")[0], consumer_secret, episode_filter=match_episode)
+        if len(info) == 1 and info[0]['subfile'] != '':
+            res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
+        return res
+
+
+class DramaFeverSeriesIE(DramaFeverIE):
+    IE_NAME = 'dramafever:series'
+    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)/\d*[a-zA-Z_][a-zA-Z0-9_]*/'
+    _TESTS = [{
+        'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
+        'info_dict': {
+            'id': '4512',
+            'title': 'Cooking with Shin',
+            'description': 'Professional chef and cooking instructor Shin Kim takes some of the delicious dishes featured in your favorite dramas and shows you how to make them right at home.',
+        },
+        'playlist_count': 4,
+    }, {
+        'url': 'http://www.dramafever.com/drama/124/IRIS/',
+        'info_dict': {
+            'id': '124',
+            'title': 'IRIS',
+            'description': 'Lee Byung Hun and Kim Tae Hee star in this powerhouse drama and ratings megahit of action, intrigue and romance.',
+        },
+        'playlist_count': 20,
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        consumer_secret = self._get_consumer_secret(series_id)
+
+        series_json = self._download_json(
+            "http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s" % (consumer_secret, series_id),
+            series_id, note='Downloading series metadata')["series"][series_id]
+
+        title = series_json["name"]
+        description = series_json["description_short"]
+
+        episodes = self._get_episodes(series_id, consumer_secret)
+        entries = []
+        for ep in episodes:
+            entries.append(self.url_result(
+                'http://www.dramafever.com%s' % ep['episode_url'], 'DramaFever', ep['guid']))
+        return self.playlist_result(entries, series_id, title, description)

From 051df9ad99d0a29d9eb984970e3e431795b6e445 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 May 2015 14:08:23 +0800
Subject: [PATCH 0157/2145] [letv/sohu] Skip tests relying on external proxies

The proxy is currently broken. See #5655 and zhuzhuor/Unblock-Youku#427
---
 youtube_dl/extractor/letv.py | 4 +---
 youtube_dl/extractor/sohu.py | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 1484ac0d2..da896caf1 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -50,9 +50,7 @@ class LetvIE(InfoExtractor):
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
-        'params': {
-            'cn_verification_proxy': 'http://proxy.uku.im:8888'
-        },
+        'skip': 'Only available in China',
     }]
 
     @staticmethod
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index eab4adfca..29bd9ce6f 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -23,9 +23,7 @@ class SohuIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'MV：Far East Movement《The Illest》',
         },
-        'params': {
-            'cn_verification_proxy': 'proxy.uku.im:8888'
-        }
+        'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
         'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',

From 137597b0ea88a92d174341b44b8f395b8897a2bf Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 15:15:28 +0800
Subject: [PATCH 0158/2145] [dramafever] Streamline code

---
 youtube_dl/extractor/dramafever.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 8fac99cc5..40787ffcd 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -80,12 +80,11 @@ class DramaFeverIE(InfoExtractor):
 
     def _get_subtitles(self, video_id, consumer_secret):
 
-        def match_episode(ep):
-            return ep['guid'] == video_id
-
         res = None
         info = self._get_episodes(
-            video_id.split(".")[0], consumer_secret, episode_filter=match_episode)
+            video_id.split(".")[0], consumer_secret,
+            episode_filter=lambda x: x['guid'] == video_id)
+
         if len(info) == 1 and info[0]['subfile'] != '':
             res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
         return res

From 2632941f327c8b013e5fbc736317fc897876ab73 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 15:53:45 +0800
Subject: [PATCH 0159/2145] [soompi] Add new extractor for tv.soompi.com

---
 youtube_dl/extractor/__init__.py |   4 +
 youtube_dl/extractor/soompi.py   | 130 +++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 youtube_dl/extractor/soompi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f8c261d5..2a5cf9547 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -482,6 +482,10 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
+from .soompi import (
+    SoompiIE,
+    SoompiShowIE,
+)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
new file mode 100644
index 000000000..5ecf40b7f
--- /dev/null
+++ b/youtube_dl/extractor/soompi.py
@@ -0,0 +1,130 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+import base64
+import xml.etree.ElementTree
+
+# Soompi uses the same subtitle encryption as crunchyroll
+from .crunchyroll import CrunchyrollIE
+
+
+class SoompiIE(CrunchyrollIE):
+    IE_NAME = 'soompi'
+    _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://tv.soompi.com/en/watch/23363',
+        'info_dict': {
+            'id': '23363',
+            'ext': 'mp4',
+            'title': 'Liar Game CM1',
+            'description': '15sec'
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _get_episodes(self, webpage, episode_filter=None):
+        episodes = json.loads(
+            self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
+        return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
+
+    def _get_subtitles(self, video_id, show_format_xml):
+        subtitles = {}
+        subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
+        subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
+
+        sub_langs = {}
+        for i in subtitle_info_nodes:
+            sub_langs[i.attrib["id"]] = i.attrib["title"]
+
+        for s in subtitle_nodes:
+            lang_code = sub_langs.get(s.attrib["id"], None)
+            if lang_code is None:
+                continue
+
+            sub_id = int(s.attrib["id"])
+            iv = base64.b64decode(s.find("iv").text)
+            data = base64.b64decode(s.find("data").text)
+            subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
+            sub_root = xml.etree.ElementTree.fromstring(subtitle)
+
+            subtitles[lang_code] = [{
+                'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
+            }, {
+                'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
+            }]
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, video_id, note="Downloading episode page",
+            errnote="Video may not be available for your location")
+        vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage)
+
+        show_meta = json.loads(
+            self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta"))
+        episodes = self._get_episodes(
+            webpage, episode_filter=lambda x: x['id'] == video_id)
+
+        title = episodes[0]["name"]
+        description = episodes[0]["description"]
+        duration = int(episodes[0]["duration"])
+        slug = show_meta["slug"]
+
+        formats = []
+        show_format_xml = None
+        for vf in vid_formats:
+            show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \
+                              % (slug, video_id, vf)
+            show_format_xml = self._download_xml(
+                show_format_url, video_id, note="Downloading q%s show xml" % vf)
+            avail_formats = self._extract_m3u8_formats(
+                show_format_xml.find('./{default}preload/stream_info/file').text,
+                video_id, ext="mp4", m3u8_id=vf, preference=int(vf))
+            formats.extend(avail_formats)
+        self._sort_formats(formats)
+
+        subtitles = self.extract_subtitles(video_id, show_format_xml)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': subtitles
+        }
+
+
+class SoompiShowIE(SoompiIE):
+    IE_NAME = 'soompi:show'
+    _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
+    _TESTS = [{
+        'url': 'http://tv.soompi.com/en/shows/liar-game',
+        'info_dict': {
+            'id': 'liar-game',
+            'title': 'Liar Game',
+            'description': 'md5:52c02bce0c1a622a95823591d0589b66',
+        },
+        'playlist_count': 14,
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, show_id, note="Downloading show page")
+        title = self._og_search_title(webpage).replace("SoompiTV | ", "")
+        description = self._og_search_description(webpage)
+
+        episodes = self._get_episodes(webpage)
+        entries = []
+        for ep in episodes:
+            entries.append(self.url_result(
+                'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
+
+        return self.playlist_result(entries, show_id, title, description)

From 0b9f7cd074786abafcd35b26db4ecb4d92814393 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 May 2015 10:01:48 +0200
Subject: [PATCH 0160/2145] release 2015.05.20

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 43fbe8b1d..a4879bd9a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -142,6 +142,7 @@
  - **Eporner**
  - **EroProfile**
  - **Escapist**
+ - **ESPN** (Currently broken)
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -338,6 +339,7 @@
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
+ - **OoyalaExternal**
  - **OpenFilm**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
@@ -451,6 +453,7 @@
  - **Spike**
  - **Sport5**
  - **SportBox**
+ - **SportBoxEmbed**
  - **SportDeutschland**
  - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
@@ -510,6 +513,8 @@
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
+ - **TV2**
+ - **TV2Article**
  - **TV4**: tv4.se and tv4play.se
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 38f00bc9b..b33385153 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.15'
+__version__ = '2015.05.20'

From 5137adb94dcce98a3c14fb3892c5c72f70ff34ea Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 16:16:10 +0800
Subject: [PATCH 0161/2145] [soompi] Switch to non-geoblocked test video

---
 youtube_dl/extractor/soompi.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
index 5ecf40b7f..4726872dc 100644
--- a/youtube_dl/extractor/soompi.py
+++ b/youtube_dl/extractor/soompi.py
@@ -14,12 +14,12 @@ class SoompiIE(CrunchyrollIE):
     IE_NAME = 'soompi'
     _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://tv.soompi.com/en/watch/23363',
+        'url': 'http://tv.soompi.com/en/watch/29235',
         'info_dict': {
-            'id': '23363',
+            'id': '29235',
             'ext': 'mp4',
-            'title': 'Liar Game CM1',
-            'description': '15sec'
+            'title': 'Episode 1096',
+            'description': '2015-05-20'
         },
         'params': {
             'skip_download': True,

From b0d619fde2b187f2b36b077a1eb11d766429f88c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 May 2015 21:28:04 +0600
Subject: [PATCH 0162/2145] [viki:channel] Extract title from JSON

---
 youtube_dl/extractor/viki.py | 40 +++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 9bdbdc3e4..fc585c299 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -23,7 +23,7 @@ class VikiIE(InfoExtractor):
     # iPad2
     _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
 
-    _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -149,7 +149,7 @@ class VikiIE(InfoExtractor):
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {
@@ -167,27 +167,35 @@ class VikiChannelIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
+    _API_BASE = 'http://api.viki.io/v4/containers'
+    _APP = '100000a'
     _PER_PAGE = 25
 
     def _real_extract(self, url):
-        show_id = self._match_id(url)
-        show_page = self._download_webpage(url, show_id, 'Download show page')
+        channel_id = self._match_id(url)
 
-        title = self._og_search_title(show_page)
-        description = self._og_search_description(show_page)
+        channel = self._download_json(
+            '%s/%s.json?app=%s' % (self._API_BASE, channel_id, self._APP),
+            channel_id, 'Downloading channel JSON')
+
+        titles = channel['titles']
+        title = titles.get('en') or titles[titles.keys()[0]]
+
+        descriptions = channel['descriptions']
+        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
 
         entries = []
-        for video_type in ['episodes', 'clips']:
-            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type, self._PER_PAGE)
-            while json_url is not None:
-                show_json = self._download_json(
-                    json_url, show_id,
-                    note='Downloading %s json page #%s' %
-                         (video_type, re.search(r'[?&]page=([0-9]+)', json_url).group(1)))
-                for video in show_json['response']:
+        for video_type in ('episodes', 'clips'):
+            page_url = '%s/%s/%s.json?app=%s&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (self._API_BASE, channel_id, video_type, self._APP, self._PER_PAGE)
+            while page_url:
+                page = self._download_json(
+                    page_url, channel_id,
+                    'Downloading %s JSON page #%s'
+                    % (video_type, re.search(r'[?&]page=([0-9]+)', page_url).group(1)))
+                for video in page['response']:
                     video_id = video['id']
                     entries.append(self.url_result(
                         'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
-                json_url = show_json['pagination']['next']
+                page_url = page['pagination']['next']
 
-        return self.playlist_result(entries, show_id, title, description)
+        return self.playlist_result(entries, channel_id, title, description)

From 1a83c731bd58ed85f6f7695cee9c88d09a224bc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 01:44:05 +0600
Subject: [PATCH 0163/2145] [viki] Switch extraction to API

---
 youtube_dl/extractor/viki.py | 230 +++++++++++++++++++++--------------
 1 file changed, 142 insertions(+), 88 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fc585c299..234649ca8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,29 +1,64 @@
 from __future__ import unicode_literals
 
 import re
+import time
+import hmac
+import hashlib
 
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
-    unescapeHTML,
-    unified_strdate,
-    US_RATINGS,
-    determine_ext,
-    mimetype2ext,
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
 )
 from .common import InfoExtractor
 
 
-class VikiIE(InfoExtractor):
+class VikiBaseIE(InfoExtractor):
+    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
+    _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
+
+    _APP = '65535a'
+    _APP_VERSION = '2.2.5.1428709186'
+    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
+
+    def _prepare_call(self, path, timestamp=None):
+        path += '?' if '?' not in path else '&'
+        if not timestamp:
+            timestamp = int(time.time())
+        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        sig = hmac.new(
+            self._APP_SECRET.encode('ascii'),
+            query.encode('ascii'),
+            hashlib.sha1
+        ).hexdigest()
+        return self._API_URL_TEMPLATE % (query, sig)
+
+    def _call_api(self, path, video_id, note, timestamp=None):
+        resp = self._download_json(
+            self._prepare_call(path, timestamp), video_id, note)
+
+        error = resp.get('error')
+        if error:
+            if error == 'invalid timestamp':
+                resp = self._download_json(
+                    self._prepare_call(path, int(resp['current_timestamp'])),
+                    video_id, '%s (retry)' % note)
+                error = resp.get('error')
+            if error:
+                self._raise_error(resp['error'])
+
+        return resp
+
+    def _raise_error(self, error):
+        raise ExtractorError(
+            '%s returned error: %s' % (self.IE_NAME, error),
+            expected=True)
+
+
+class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
-
-    # iPad2
-    _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
-
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:videos|player)/(?P<id>[0-9]+v)'
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -37,115 +72,134 @@ class VikiIE(InfoExtractor):
         },
         'skip': 'Blocked in the US',
     }, {
+        # clip
         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
-        'md5': 'ca6493e6f0a6ec07da9aa8d6304b4b2c',
+        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
         'info_dict': {
             'id': '1067139v',
             'ext': 'mp4',
+            'title': "'The Avengers: Age of Ultron' Press Conference",
             'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
+            'duration': 352,
+            'timestamp': 1430380829,
             'upload_date': '20150430',
-            'title': '\'The Avengers: Age of Ultron\' Press Conference',
+            'uploader': 'Arirang TV',
+            'like_count': int,
+            'age_limit': 0,
         }
     }, {
         'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
         'info_dict': {
             'id': '1048879v',
             'ext': 'mp4',
-            'upload_date': '20140820',
-            'description': 'md5:54ff56d51bdfc7a30441ec967394e91c',
             'title': 'Ankhon Dekhi',
+            'duration': 6512,
+            'timestamp': 1408532356,
+            'upload_date': '20140820',
+            'uploader': 'Spuul',
+            'like_count': int,
+            'age_limit': 13,
         },
         'params': {
-            # requires ffmpeg
+            # m3u8 download
             'skip_download': True,
         }
+    }, {
+        # episode
+        'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
+        'md5': '190f3ef426005ba3a080a63325955bc3',
+        'info_dict': {
+            'id': '44699v',
+            'ext': 'mp4',
+            'title': 'Boys Over Flowers - Episode 1',
+            'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
+            'duration': 4155,
+            'timestamp': 1270496524,
+            'upload_date': '20100405',
+            'uploader': 'group8',
+            'like_count': int,
+            'age_limit': 13,
+        }
+    }, {
+        'url': 'http://www.viki.com/player/44699v',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        streams = self._call_api(
+            'videos/%s/streams.json' % video_id, video_id,
+            'Downloading video streams JSON')
 
-        uploader_m = re.search(
-            r'<strong>Broadcast Network: </strong>\s*([^<]*)<', webpage)
-        if uploader_m is None:
-            uploader = None
-        else:
-            uploader = uploader_m.group(1).strip()
+        formats = []
+        for format_id, stream_dict in streams.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            for protocol, format_dict in stream_dict.items():
+                if format_id == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                else:
+                    formats.append({
+                        'url': format_dict['url'],
+                        'format_id': '%s-%s' % (format_id, protocol),
+                        'height': height,
+                    })
+        self._sort_formats(formats)
 
-        rating_str = self._html_search_regex(
-            r'<strong>Rating: </strong>\s*([^<]*)<', webpage,
-            'rating information', default='').strip()
-        age_limit = US_RATINGS.get(rating_str)
+        video = self._call_api(
+            'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
-        req = compat_urllib_request.Request(
-            'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id)
-        req.add_header('User-Agent', self._USER_AGENT)
-        info_webpage = self._download_webpage(
-            req, video_id, note='Downloading info page')
-        err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
-        if err_msg:
-            if 'not available in your region' in err_msg:
-                raise ExtractorError(
-                    'Video %s is blocked from your location.' % video_id,
-                    expected=True)
-            else:
-                raise ExtractorError('Viki said: %s %s' % (err_msg, url))
-        mobj = re.search(
-            r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
-        if not mobj:
-            raise ExtractorError('Unable to find video URL')
-        video_url = unescapeHTML(mobj.group('url'))
-        video_ext = mimetype2ext(mobj.group('mime_type'))
+        title = None
+        titles = video.get('titles')
+        if titles:
+            title = titles.get('en') or titles[titles.keys()[0]]
+        if not title:
+            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
+            container_titles = video.get('container', {}).get('titles')
+            if container_titles:
+                container_title = container_titles.get('en') or container_titles[titles.keys()[0]]
+                title = '%s - %s' % (container_title, title)
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(
-                video_url, video_id, ext=video_ext)
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': video_ext,
-            }]
+        descriptions = video.get('descriptions')
+        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
 
-        upload_date_str = self._html_search_regex(
-            r'"created_at":"([^"]+)"', info_webpage, 'upload date')
-        upload_date = (
-            unified_strdate(upload_date_str)
-            if upload_date_str is not None
-            else None
-        )
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('created_at'))
+        uploader = video.get('author')
+        like_count = int_or_none(video.get('likes', {}).get('count'))
+        age_limit = parse_age_limit(video.get('rating'))
 
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, info_webpage)
+        thumbnails = []
+        for thumbnail_id, thumbnail in video.get('images', {}).items():
+            thumbnails.append({
+                'id': thumbnail_id,
+                'url': thumbnail.get('url'),
+            })
+
+        subtitles = {}
+        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
+            subtitles[subtitle_lang] = [{
+                'ext': subtitles_format,
+                'url': self._prepare_call(
+                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
+            } for subtitles_format in ('srt', 'vtt')]
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
             'description': description,
-            'thumbnail': thumbnail,
-            'age_limit': age_limit,
+            'duration': duration,
+            'timestamp': timestamp,
             'uploader': uploader,
-            'subtitles': video_subtitles,
-            'upload_date': upload_date,
+            'like_count': like_count,
+            'age_limit': age_limit,
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
         }
 
-    def _get_subtitles(self, video_id, info_webpage):
-        res = {}
-        for sturl_html in re.findall(r'<track src="([^"]+)"', info_webpage):
-            sturl = unescapeHTML(sturl_html)
-            m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl)
-            if not m:
-                continue
-            res[m.group('lang')] = [{
-                'url': compat_urlparse.urljoin('http://www.viki.com', sturl),
-                'ext': 'vtt',
-            }]
-        return res
-
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'

From ac20d95f9766aa130748aac07fa90ee5dfa566d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 01:56:02 +0600
Subject: [PATCH 0164/2145] [viki] Add support for youtube externals

---
 youtube_dl/extractor/viki.py | 70 ++++++++++++++++++++++++------------
 1 file changed, 48 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 234649ca8..68d5cac6e 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -120,6 +120,23 @@ class VikiIE(VikiBaseIE):
             'like_count': int,
             'age_limit': 13,
         }
+    }, {
+        # youtube external
+        'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
+        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
+        'info_dict': {
+            'id': '50562v',
+            'ext': 'mp4',
+            'title': 'Poor Nastya [COMPLETE] - Episode 1',
+            'description': '',
+            'duration': 607,
+            'timestamp': 1274949505,
+            'upload_date': '20101213',
+            'uploader': 'ad14065n',
+            'uploader_id': 'ad14065n',
+            'like_count': int,
+            'age_limit': 13,
+        }
     }, {
         'url': 'http://www.viki.com/player/44699v',
         'only_matching': True,
@@ -128,26 +145,6 @@ class VikiIE(VikiBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        streams = self._call_api(
-            'videos/%s/streams.json' % video_id, video_id,
-            'Downloading video streams JSON')
-
-        formats = []
-        for format_id, stream_dict in streams.items():
-            height = self._search_regex(
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
-            for protocol, format_dict in stream_dict.items():
-                if format_id == 'm3u8':
-                    formats = self._extract_m3u8_formats(
-                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
-                else:
-                    formats.append({
-                        'url': format_dict['url'],
-                        'format_id': '%s-%s' % (format_id, protocol),
-                        'height': height,
-                    })
-        self._sort_formats(formats)
-
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
@@ -186,7 +183,7 @@ class VikiIE(VikiBaseIE):
                     'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
             } for subtitles_format in ('srt', 'vtt')]
 
-        return {
+        result = {
             'id': video_id,
             'title': title,
             'description': description,
@@ -196,10 +193,39 @@ class VikiIE(VikiBaseIE):
             'like_count': like_count,
             'age_limit': age_limit,
             'thumbnails': thumbnails,
-            'formats': formats,
             'subtitles': subtitles,
         }
 
+        streams = self._call_api(
+            'videos/%s/streams.json' % video_id, video_id,
+            'Downloading video streams JSON')
+
+        if 'external' in streams:
+            result.update({
+                '_type': 'url_transparent',
+                'url': streams['external']['url'],
+            })
+            return result
+
+        formats = []
+        for format_id, stream_dict in streams.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            for protocol, format_dict in stream_dict.items():
+                if format_id == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                else:
+                    formats.append({
+                        'url': format_dict['url'],
+                        'format_id': '%s-%s' % (format_id, protocol),
+                        'height': height,
+                    })
+        self._sort_formats(formats)
+
+        result['formats'] = formats
+        return result
+
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'

From bc56355ec6bc823fe96e31688cd3123dc18ae627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:08:13 +0600
Subject: [PATCH 0165/2145] [viki:channel] Switch to API

---
 youtube_dl/extractor/viki.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 68d5cac6e..071e280fb 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -4,6 +4,7 @@ import re
 import time
 import hmac
 import hashlib
+import itertools
 
 from ..utils import (
     ExtractorError,
@@ -227,7 +228,7 @@ class VikiIE(VikiBaseIE):
         return result
 
 
-class VikiChannelIE(InfoExtractor):
+class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
     _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
@@ -247,16 +248,15 @@ class VikiChannelIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
-    _API_BASE = 'http://api.viki.io/v4/containers'
-    _APP = '100000a'
+
     _PER_PAGE = 25
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        channel = self._download_json(
-            '%s/%s.json?app=%s' % (self._API_BASE, channel_id, self._APP),
-            channel_id, 'Downloading channel JSON')
+        channel = self._call_api(
+            'containers/%s.json' % channel_id, channel_id,
+            'Downloading channel JSON')
 
         titles = channel['titles']
         title = titles.get('en') or titles[titles.keys()[0]]
@@ -266,16 +266,16 @@ class VikiChannelIE(InfoExtractor):
 
         entries = []
         for video_type in ('episodes', 'clips'):
-            page_url = '%s/%s/%s.json?app=%s&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (self._API_BASE, channel_id, video_type, self._APP, self._PER_PAGE)
-            while page_url:
-                page = self._download_json(
-                    page_url, channel_id,
-                    'Downloading %s JSON page #%s'
-                    % (video_type, re.search(r'[?&]page=([0-9]+)', page_url).group(1)))
+            for page_num in itertools.count(1):
+                page = self._call_api(
+                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
+                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
+                    'Downloading %s JSON page #%d' % (video_type, page_num))
                 for video in page['response']:
                     video_id = video['id']
                     entries.append(self.url_result(
-                        'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
-                page_url = page['pagination']['next']
+                        'http://www.viki.com/videos/%s' % video_id, 'Viki'))
+                if not page['pagination']['next']:
+                    break
 
         return self.playlist_result(entries, channel_id, title, description)

From d01924f48810db69d572bc121ab98021f04ac957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:30:04 +0600
Subject: [PATCH 0166/2145] [viki:channel] Extend matching URLs and extract
 movies

---
 youtube_dl/extractor/viki.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 071e280fb..3acb481f9 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -230,7 +230,7 @@ class VikiIE(VikiBaseIE):
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:tv|news|movies|artists)/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {
@@ -247,6 +247,15 @@ class VikiChannelIE(VikiBaseIE):
             'description': 'md5:05bf5471385aa8b21c18ad450e350525',
         },
         'playlist_count': 127,
+    }, {
+        'url': 'http://www.viki.com/news/24569c-showbiz-korea',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viki.com/artists/2141c-shinee',
+        'only_matching': True,
     }]
 
     _PER_PAGE = 25
@@ -265,7 +274,7 @@ class VikiChannelIE(VikiBaseIE):
         description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
 
         entries = []
-        for video_type in ('episodes', 'clips'):
+        for video_type in ('episodes', 'clips', 'movies'):
             for page_num in itertools.count(1):
                 page = self._call_api(
                     'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'

From 4d8ee01389c4229f14fad45f0aa7b033a2509aef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:38:43 +0600
Subject: [PATCH 0167/2145] [viki] Fix typo

---
 youtube_dl/extractor/viki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 3acb481f9..0ec8ef0ef 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -157,7 +157,7 @@ class VikiIE(VikiBaseIE):
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
             container_titles = video.get('container', {}).get('titles')
             if container_titles:
-                container_title = container_titles.get('en') or container_titles[titles.keys()[0]]
+                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                 title = '%s - %s' % (container_title, title)
 
         descriptions = video.get('descriptions')

From 4d2f42361e02bb67de7c2017c6817b46ff3b2bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 May 2015 11:42:20 +0200
Subject: [PATCH 0168/2145] [viki] remove unused import

---
 youtube_dl/extractor/viki.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0ec8ef0ef..fe7229952 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import time
 import hmac
 import hashlib

From e7752cd57853718c6875b02517613d14c4c7221d Mon Sep 17 00:00:00 2001
From: frenchy1983 <frenchy1983@zoho.com>
Date: Thu, 21 May 2015 11:47:16 +0200
Subject: [PATCH 0169/2145] [TNAFlix] Allow dot (and more) in cat_id and
 display_id

URLs with dots were raising a "UnsupportedError: Unsupported URL" error.
---
 youtube_dl/extractor/tnaflix.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index d48cbbf14..725edd3c7 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,23 +10,23 @@ from ..utils import (
 
 
 class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[\w-]+)/(?P<display_id>[\w-]+)/video(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[^/]+)/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
     _TEST = {
-        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+        'md5': '6c431ea56756497e227fb3f01a687869',
         'info_dict': {
-            'id': '553878',
-            'display_id': 'Carmella-Decesare-striptease',
+            'id': '358632',
+            'display_id': 'bunzHD-Ms.Donk',
             'ext': 'mp4',
-            'title': 'Carmella Decesare - striptease',
-            'description': '',
+            'title': 'bunzHD Ms.Donk',
+            'description': 'bubble booty ebony teen goddess Ms.Donk has a firm ass and acts like she is shy but really she is a freak in the sheets watch her 20 min XX rated vid at bunzHD.com click on the catalog link',
             'thumbnail': 're:https?://.*\.jpg$',
-            'duration': 91,
+            'duration': 394,
             'age_limit': 18,
         }
     }

From 6ad9cb224a7d9156109fe0b0100d277b954063d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 May 2015 12:02:53 +0200
Subject: [PATCH 0170/2145] [mitele] It now uses m3u8 (#5764)

It should also be possible to use Adobe HDS, but it would require more work.
---
 youtube_dl/extractor/mitele.py    | 9 +++++++--
 youtube_dl/extractor/telecinco.py | 4 ++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index d8897eb90..7091f3335 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -20,7 +20,6 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': '6a75fe9d0d3275bead0cb683c616fddb',
         'info_dict': {
             'id': '0fce117d',
             'ext': 'mp4',
@@ -29,6 +28,10 @@ class MiTeleIE(InfoExtractor):
             'display_id': 'programa-144',
             'duration': 2913,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -56,12 +59,14 @@ class MiTeleIE(InfoExtractor):
             episode,
             transform_source=strip_jsonp
         )
+        formats = self._extract_m3u8_formats(
+            token_info['tokenizedUrl'], episode, ext='mp4')
 
         return {
             'id': embed_data['videoId'],
             'display_id': episode,
             'title': info_el.find('title').text,
-            'url': token_info['tokenizedUrl'],
+            'formats': formats,
             'description': get_element_by_attribute('class', 'text', webpage),
             'thumbnail': info_el.find('thumb').text,
             'duration': parse_duration(info_el.find('duration').text),
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 251a68680..a0c744fd1 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -16,6 +16,10 @@ class TelecincoIE(MiTeleIE):
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,

From 663004ac2b001b9be03bd951d539a62cf83c58ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 22:06:25 +0600
Subject: [PATCH 0171/2145] [options] Clarify `--metadata-from-title`
 additional templates

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 22dbc3aec..dd07266b7 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -713,7 +713,7 @@ def parseOpts(overrideArguments=None):
         help='Parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
              'the parsed parameters replace existing values. '
-             'Additional templates: %(album), %(artist). '
+             'Additional templates: %(album)s, %(artist)s. '
              'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise"')
     postproc.add_option(

From 53de95da5e40aa1a465668977e507ccc914099f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 22:27:22 +0600
Subject: [PATCH 0172/2145] [viki] Extend _VALID_URLs

---
 youtube_dl/extractor/viki.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fe7229952..7f2fb1ca8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -15,6 +15,7 @@ from .common import InfoExtractor
 
 
 class VikiBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
     _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
     _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
 
@@ -58,7 +59,7 @@ class VikiBaseIE(InfoExtractor):
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:videos|player)/(?P<id>[0-9]+v)'
+    _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -229,7 +230,7 @@ class VikiIE(VikiBaseIE):
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:tv|news|movies|artists)/(?P<id>[0-9]+c)'
+    _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {

From 5cd47a5e4f54033bcf6d80908e00eff4c75a51c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 23:58:46 +0600
Subject: [PATCH 0173/2145] [videott] Fix for python 3.2

---
 youtube_dl/extractor/videott.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index ececc7ee0..591024ead 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -43,7 +43,7 @@ class VideoTtIE(InfoExtractor):
 
         formats = [
             {
-                'url': base64.b64decode(res['u']).decode('utf-8'),
+                'url': base64.b64decode(res['u'].encode('utf-8')).decode('utf-8'),
                 'ext': 'flv',
                 'format_id': res['l'],
             } for res in settings['res'] if res['u']

From 06947add03b6b619292812a771993d3365b0e7e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:03:47 +0600
Subject: [PATCH 0174/2145] [chilloutzone] Fix for python 3.2

---
 youtube_dl/extractor/chilloutzone.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dl/extractor/chilloutzone.py
index c922f6959..0206d96db 100644
--- a/youtube_dl/extractor/chilloutzone.py
+++ b/youtube_dl/extractor/chilloutzone.py
@@ -57,7 +57,7 @@ class ChilloutzoneIE(InfoExtractor):
 
         base64_video_info = self._html_search_regex(
             r'var cozVidData = "(.+?)";', webpage, 'video data')
-        decoded_video_info = base64.b64decode(base64_video_info).decode("utf-8")
+        decoded_video_info = base64.b64decode(base64_video_info.encode('utf-8')).decode('utf-8')
         video_info_dict = json.loads(decoded_video_info)
 
         # get video information from dict

From 878563c847fa5248eedbd44187536dec04643eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:06:10 +0600
Subject: [PATCH 0175/2145] [aes] Fix for python 3.2

---
 youtube_dl/aes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 07224d508..7817adcfd 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -152,7 +152,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     """
     NONCE_LENGTH_BYTES = 8
 
-    data = bytes_to_intlist(base64.b64decode(data))
+    data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
     password = bytes_to_intlist(password.encode('utf-8'))
 
     key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))

From afe8b594be53161f68189e15a65b4e9c6eba0b35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:09:15 +0600
Subject: [PATCH 0176/2145] [rtve.es:alacarta] Fix for python 3.2

---
 youtube_dl/extractor/rtve.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 849300140..82cd98ac7 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 def _decrypt_url(png):
-    encrypted_data = base64.b64decode(png)
+    encrypted_data = base64.b64decode(png.encode('utf-8'))
     text_index = encrypted_data.find(b'tEXt')
     text_chunk = encrypted_data[text_index - 4:]
     length = struct_unpack('!I', text_chunk[:4])[0]

From 43150d7ac36efda7bc60c694b8a18e1f720da04b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:10:05 +0600
Subject: [PATCH 0177/2145] [shared] Fix for python 3.2

---
 youtube_dl/extractor/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 26ced716e..9f3e944e7 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -47,7 +47,7 @@ class SharedIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'data-url="([^"]+)"', video_page, 'video URL')
         title = base64.b64decode(self._html_search_meta(
-            'full:title', webpage, 'title')).decode('utf-8')
+            'full:title', webpage, 'title').encode('utf-8')).decode('utf-8')
         filesize = int_or_none(self._html_search_meta(
             'full:size', webpage, 'file size', fatal=False))
         thumbnail = self._html_search_regex(

From 0459432d962bf358566340eed00f6c1c56b7b732 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:10:53 +0600
Subject: [PATCH 0178/2145] [shared] Fix for python 3.2

---
 youtube_dl/extractor/tutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index 4de0aac52..fad720b68 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -26,7 +26,7 @@ class TutvIE(InfoExtractor):
 
         data_content = self._download_webpage(
             'http://tu.tv/flvurl.php?codVideo=%s' % internal_id, video_id, 'Downloading video info')
-        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0]).decode('utf-8')
+        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0].encode('utf-8')).decode('utf-8')
 
         return {
             'id': internal_id,

From 77d9cb2f04462677f2a36f487c20e7a7992a0a32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:45:33 +0600
Subject: [PATCH 0179/2145] [sportbox] Fix extraction

---
 youtube_dl/extractor/sportbox.py | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 8686f9d11..2ab3489e4 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
-    parse_iso8601,
+    unified_strdate,
 )
 
 
@@ -20,11 +20,9 @@ class SportBoxIE(InfoExtractor):
             'id': '80822',
             'ext': 'mp4',
             'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-            'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+            'description': 'md5:3d72dc4a006ab6805d82f037fdc637ad',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'timestamp': 1411896237,
             'upload_date': '20140928',
-            'duration': 4846,
         },
         'params': {
             # m3u8 download
@@ -48,17 +46,13 @@ class SportBoxIE(InfoExtractor):
             r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
 
         title = self._html_search_regex(
-            r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
-        description = self._html_search_regex(
-            r'(?s)<div itemprop="description">(.+?)</div>',
-            webpage, 'description', fatal=False)
+            [r'"nodetitle"\s*:\s*"([^"]+)"', r'class="node-header_{1,2}title">([^<]+)'],
+            webpage, 'title')
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
         thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate">([^<]+)</span>',
-            webpage, 'timestamp', fatal=False))
-        duration = parse_duration(self._html_search_regex(
-            r'<meta itemprop="duration" content="PT([^"]+)">',
-            webpage, 'duration', fatal=False))
+        upload_date = unified_strdate(self._html_search_meta(
+            'dateCreated', webpage, 'upload date'))
 
         return {
             '_type': 'url_transparent',
@@ -67,8 +61,7 @@ class SportBoxIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
+            'upload_date': upload_date,
         }
 
 

From 8a278a1d7ef6134a5ac6b7dd31e3458d05f71225 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 May 2015 13:26:50 +0800
Subject: [PATCH 0180/2145] [nba] Fix duration extraction (fixes #5777)

---
 youtube_dl/extractor/nba.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 862b706bf..944096e1c 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -22,6 +22,18 @@ class NBAIE(InfoExtractor):
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
+    }, {
+        'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'info_dict': {
+            'id': '0041400301-cle-atl-recap.nba',
+            'ext': 'mp4',
+            'title': 'NBA GAME TIME | Video: Hawks vs. Cavaliers Game 1',
+            'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
+            'duration': 228,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
@@ -35,8 +47,12 @@ class NBAIE(InfoExtractor):
             self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
 
         description = self._og_search_description(webpage)
-        duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration'))
+        duration_str = self._html_search_meta(
+            'duration', webpage, 'duration', default=None)
+        if not duration_str:
+            duration_str = self._html_search_regex(
+                r'Duration:</b>\s*(\d+:\d+)', webpage, 'duration', fatal=False)
+        duration = parse_duration(duration_str)
 
         return {
             'id': shortened_video_id,

From ed5a637d62e8ede4a8cef75df4e5f341e3c667a1 Mon Sep 17 00:00:00 2001
From: frenchy1983 <frenchy1983@zoho.com>
Date: Fri, 22 May 2015 09:29:35 +0200
Subject: [PATCH 0181/2145] [TNAFlix] Restore test

See dstftw's comment in #5772
---
 youtube_dl/extractor/empflix.py | 26 ++++++++++++++------------
 youtube_dl/extractor/tnaflix.py | 32 +++++++++++++++++++-------------
 2 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 70f8efe27..0dc947c1d 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -10,16 +10,18 @@ class EMPFlixIE(TNAFlixIE):
     _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
-    _TEST = {
-        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
-        'info_dict': {
-            'id': '33051',
-            'display_id': 'Amateur-Finger-Fuck',
-            'ext': 'mp4',
-            'title': 'Amateur Finger Fuck',
-            'description': 'Amateur solo finger fucking.',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'age_limit': 18,
+    _TESTS = [
+        {
+            'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
+            'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+            'info_dict': {
+                'id': '33051',
+                'display_id': 'Amateur-Finger-Fuck',
+                'ext': 'mp4',
+                'title': 'Amateur Finger Fuck',
+                'description': 'Amateur solo finger fucking.',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'age_limit': 18,
+            }
         }
-    }
+    ]
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 725edd3c7..79496039d 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -16,20 +16,26 @@ class TNAFlixIE(InfoExtractor):
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
-    _TEST = {
-        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-        'md5': '6c431ea56756497e227fb3f01a687869',
-        'info_dict': {
-            'id': '358632',
-            'display_id': 'bunzHD-Ms.Donk',
-            'ext': 'mp4',
-            'title': 'bunzHD Ms.Donk',
-            'description': 'bubble booty ebony teen goddess Ms.Donk has a firm ass and acts like she is shy but really she is a freak in the sheets watch her 20 min XX rated vid at bunzHD.com click on the catalog link',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'duration': 394,
-            'age_limit': 18,
+    _TESTS = [
+        {
+            'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+            'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+            'info_dict': {
+                'id': '553878',
+                'display_id': 'Carmella-Decesare-striptease',
+                'ext': 'mp4',
+                'title': 'Carmella Decesare - striptease',
+                'description': '',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'duration': 91,
+                'age_limit': 18,
+            }
+        },
+        {
+            'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+            'matching_only': True,
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From ba6454761687e099f960b50cb50a9b87f4ec6d17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 11:35:09 +0200
Subject: [PATCH 0182/2145] [sportbox] Remove unused import

---
 youtube_dl/extractor/sportbox.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 2ab3489e4..86d509ae5 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    parse_duration,
     unified_strdate,
 )
 

From 79979c689713fd28e8fdf08bd71eecb6798f23d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 16:14:55 +0200
Subject: [PATCH 0183/2145] Clarify that --dump-pages encodes the pages using
 base64 (#5781)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index dd07266b7..5a2315bd9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -537,7 +537,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
-        help='Print downloaded pages to debug problems (very verbose)')
+        help='Print downloaded pages encoded using base64 to debug problems (very verbose)')
     verbosity.add_option(
         '--write-pages',
         action='store_true', dest='write_pages', default=False,

From 69e0f1b445388e4b6f45868d53780d6f8937f56e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 00:08:10 +0600
Subject: [PATCH 0184/2145] Credit @ping for viki:channel, qqmusic:toplist

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 267b8da1e..ebed7ebb3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -124,3 +124,4 @@ Mohammad Teimori Pabandi
 Roman Le Négrate
 Matthias Küch
 Julian Richen
+Ping O.

From 685c74d315a54154c5a1d9ecee8b212dbee94bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 01:01:47 +0600
Subject: [PATCH 0185/2145] [rutv] Extend embed URL (Closes #5782)

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 55604637d..d9df06861 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -104,7 +104,7 @@ class RUTVIE(InfoExtractor):
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.rutv\.ru/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From d386878af96f368ba4c2fc8bc9b078a69b79fdf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 21:25:53 +0600
Subject: [PATCH 0186/2145] [prosiebensat1] Add support for .at domain names
 (Closes #5786)

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 7cc799664..255d4abc1 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -17,7 +17,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|ran|the-voice-of-germany)\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {

From abca34cbc04693662d913e19634c06c214a237f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:04:02 +0800
Subject: [PATCH 0187/2145] [cnn] Relax _VALID_URL again (fixes #5737)

The problem is the same as test:CNN_1, so I didn't add the test case
---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 5efc5f4fe..3b1bd4033 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z]{3,5})(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',

From 9bf87ae3aaac81df3efb92fd0a3247ccb522de2a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:36:47 +0800
Subject: [PATCH 0188/2145] [nextmedia] Merge AppleDailyRealtimeNewsIE and
 AppleDailyAnimationNewsIE

---
 youtube_dl/extractor/__init__.py  |  3 +--
 youtube_dl/extractor/nextmedia.py | 37 +++++++++++++------------------
 2 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 24efb7ce5..8bb3926a0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -338,8 +338,7 @@ from .newstube import NewstubeIE
 from .nextmedia import (
     NextMediaIE,
     NextMediaActionNewsIE,
-    AppleDailyRealtimeNewsIE,
-    AppleDailyAnimationNewsIE
+    AppleDailyIE,
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index 02dba4ef6..c75ccafc7 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -89,8 +89,8 @@ class NextMediaActionNewsIE(NextMediaIE):
         return self._extract_from_nextmedia_page(news_id, url, article_page)
 
 
-class AppleDailyRealtimeNewsIE(NextMediaIE):
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+class AppleDailyIE(NextMediaIE):
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(animation|realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -99,7 +99,7 @@ class AppleDailyRealtimeNewsIE(NextMediaIE):
             'ext': 'mp4',
             'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:b23787119933404ce515c6356a8c355c',
+            'description': 'md5:2acd430e59956dc47cd7f67cb3c003f4',
             'upload_date': '20150128',
         }
     }, {
@@ -110,26 +110,10 @@ class AppleDailyRealtimeNewsIE(NextMediaIE):
             'ext': 'mp4',
             'title': '不滿被踩腳　山東兩大媽一路打下車',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:2648aaf6fc4f401f6de35a91d111aa1d',
+            'description': 'md5:175b4260c1d7c085993474217e4ab1b4',
             'upload_date': '20150128',
         }
-    }]
-
-    _URL_PATTERN = r'\{url: \'(.+)\'\}'
-
-    def _fetch_title(self, page):
-        return self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title')
-
-    def _fetch_thumbnail(self, page):
-        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
-
-    def _fetch_timestamp(self, page):
-        return None
-
-
-class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
-    _VALID_URL = 'http://www.appledaily.com.tw/animation/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
-    _TESTS = [{
+    }, {
         'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
         'md5': '03df296d95dedc2d5886debbb80cb43f',
         'info_dict': {
@@ -156,8 +140,17 @@ class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
         ]
     }]
 
+    _URL_PATTERN = r'\{url: \'(.+)\'\}'
+
     def _fetch_title(self, page):
-        return self._html_search_meta('description', page, 'news title')
+        return (self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title', default=None) or
+                self._html_search_meta('description', page, 'news title'))
+
+    def _fetch_thumbnail(self, page):
+        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
+
+    def _fetch_timestamp(self, page):
+        return None
 
     def _fetch_description(self, page):
         return self._html_search_meta('description', page, 'news description')

From 30455ce2554d00489901a398d457fac89456fe49 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:42:01 +0800
Subject: [PATCH 0189/2145] [nextmedia] Extend and reorder _VALID_URL

---
 youtube_dl/extractor/nextmedia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c75ccafc7..d1b7cff4c 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -90,7 +90,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 
 class AppleDailyIE(NextMediaIE):
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(animation|realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -138,6 +138,9 @@ class AppleDailyIE(NextMediaIE):
         'expected_warnings': [
             'video thumbnail',
         ]
+    }, {
+        'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
+        'only_matching': True,
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From 1335c3aca8f0cbddc0c521c73579eec2b9a5643c Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 24 May 2015 01:21:18 +0600
Subject: [PATCH 0190/2145] [drtv] Improve extraction (Closes #5792)

---
 youtube_dl/extractor/drtv.py | 37 +++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index f25ab319e..baa24c6d1 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,8 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor, ExtractorError
-from ..utils import parse_iso8601
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
 
 
 class DRTVIE(InfoExtractor):
@@ -60,19 +63,31 @@ class DRTVIE(InfoExtractor):
                 restricted_to_denmark = asset['RestrictedToDenmark']
                 spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                 for link in asset['Links']:
-                    target = link['Target']
                     uri = link['Uri']
+                    target = link['Target']
                     format_id = target
-                    preference = -1 if target == 'HDS' else -2
+                    preference = None
                     if spoken_subtitles:
-                        preference -= 2
+                        preference = -1
                         format_id += '-spoken-subtitles'
-                    formats.append({
-                        'url': uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43' if target == 'HDS' else uri,
-                        'format_id': format_id,
-                        'ext': link['FileFormat'],
-                        'preference': preference,
-                    })
+                    if target == 'HDS':
+                        formats.extend(self._extract_f4m_formats(
+                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
+                            video_id, preference, f4m_id=format_id))
+                    elif target == 'HLS':
+                        formats.extend(self._extract_m3u8_formats(
+                            uri, video_id, 'mp4', preference=preference,
+                            m3u8_id=format_id))
+                    else:
+                        bitrate = link.get('Bitrate')
+                        if bitrate:
+                            format_id += '-%s' % bitrate
+                        formats.append({
+                            'url': uri,
+                            'format_id': format_id,
+                            'tbr': bitrate,
+                            'ext': link.get('FileFormat'),
+                        })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
                     LANGS = {

From 71646e465348b25962a15f9a567f134514bde30a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 04:14:01 +0600
Subject: [PATCH 0191/2145] [YoutubeDL] Initialize `files_to_delete` (Closes
 #5797)

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 58b34e087..d1953c18f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1527,6 +1527,7 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
+            files_to_delete = []
             try:
                 files_to_delete, info = pp.run(info)
             except PostProcessingError as e:

From 1807ae22dd93646ea4d7ba4bd28087bf1ef4857c Mon Sep 17 00:00:00 2001
From: WassimAttar <wassim.attar@free.fr>
Date: Sun, 24 May 2015 10:37:05 +0200
Subject: [PATCH 0192/2145] chmod error

After installing youtube-dl with this method
    sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
    sudo chmod a+xr /usr/local/bin/youtube-dl
When i try to use it, i get this error
    python: can't open file '/usr/local/bin/youtube-dl': [Errno 13] Permission denied

The correct chmod is a+xr
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3d9436456..a29cccb3f 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ youtube-dl - download videos from youtube.com or other video platforms
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
     sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
-    sudo chmod a+x /usr/local/bin/youtube-dl
+    sudo chmod a+xr /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
-    sudo chmod a+x /usr/local/bin/youtube-dl
+    sudo chmod a+xr /usr/local/bin/youtube-dl
 
 Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
 

From 23905927e19280d9217ecad377ef26ea9d5793fe Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 24 May 2015 18:32:04 +0600
Subject: [PATCH 0193/2145] [README.md] Keep more idiomatic rwx order

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index a29cccb3f..e51bb5343 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ youtube-dl - download videos from youtube.com or other video platforms
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
     sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
-    sudo chmod a+xr /usr/local/bin/youtube-dl
+    sudo chmod a+rx /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
-    sudo chmod a+xr /usr/local/bin/youtube-dl
+    sudo chmod a+rx /usr/local/bin/youtube-dl
 
 Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
 

From abac15f3c6915d176c37f7aa748b8a0f03db82a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 19:08:22 +0600
Subject: [PATCH 0194/2145] [tnaflix] Do not capture `cat_id`

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 79496039d..59af9aba0 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[^/]+)/(?P<display_id>[^/]+)/video(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'

From 34fb7e46ad3fa1a04635fa4876401aac881bb39b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 19:10:03 +0600
Subject: [PATCH 0195/2145] [empflix] Relax _VALID_URL

---
 youtube_dl/extractor/empflix.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 0dc947c1d..9a5a8f4bb 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -4,7 +4,7 @@ from .tnaflix import TNAFlixIE
 
 
 class EMPFlixIE(TNAFlixIE):
-    _VALID_URL = r'^https?://www\.empflix\.com/videos/(?P<display_id>[0-9a-zA-Z-]+)-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
 
     _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
     _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
@@ -23,5 +23,9 @@ class EMPFlixIE(TNAFlixIE):
                 'thumbnail': 're:https?://.*\.jpg$',
                 'age_limit': 18,
             }
+        },
+        {
+            'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
+            'matching_only': True,
         }
     ]

From d78c834ead934e5532d2f5bc221bb11eedaef0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 20:04:13 +0600
Subject: [PATCH 0196/2145] [karrierevideos] Improve and simplify

---
 youtube_dl/extractor/karrierevideos.py | 96 +++++++++++++++++++-------
 1 file changed, 71 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index a05e8ab76..bed94bc93 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -1,50 +1,96 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    fix_xml_ampersands,
+    float_or_none,
+    xpath_with_ns,
+    xpath_text,
+)
 
 
 class KarriereVideosIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P<id>[a-z-]+)'
-    _TEST = {
+    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
         'info_dict': {
-            'id': 'altenpflegerin',
-            'ext': 'mp4',
+            'id': '32c91',
+            'ext': 'flv',
             'title': 'AltenpflegerIn',
-            'thumbnail': 're:^http://.*\.png\?v=[0-9]+',
-            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2'
+            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2',
+            'thumbnail': 're:^http://.*\.png',
         },
         'params': {
-            'skip_download': 'requires rtmpdump'
+            # rtmp download
+            'skip_download': True,
         }
-    }
+    }, {
+        # broken ampersands
+        'url': 'http://www.karrierevideos.at/orientierung/vaeterkarenz-und-neue-chancen-fuer-muetter-baby-was-nun',
+        'info_dict': {
+            'id': '5sniu',
+            'ext': 'flv',
+            'title': 'Väterkarenz und neue Chancen für Mütter - "Baby - was nun?"',
+            'description': 'md5:97092c6ad1fd7d38e9d6a5fdeb2bcc33',
+            'thumbnail': 're:^http://.*\.png',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
+        title = (self._html_search_meta('title', webpage, default=None) or
+                 self._search_regex(r'<h1 class="title">([^<]+)</h1>'))
+
+        video_id = self._search_regex(
+            r'/config/video/(.+?)\.xml', webpage, 'video id')
+        playlist = self._download_xml(
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id,
+            video_id, transform_source=fix_xml_ampersands)
+
+        NS_MAP = {
+            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
+        }
+
+        def ns(path):
+            return xpath_with_ns(path, NS_MAP)
+
+        item = playlist.find('./tracklist/item')
+        video_file = xpath_text(
+            item, ns('./jwplayer:file'), 'video url', fatal=True)
+        streamer = xpath_text(
+            item, ns('./jwplayer:streamer'), 'streamer', fatal=True)
+
+        uploader = xpath_text(
+            item, ns('./jwplayer:author'), 'uploader')
+        duration = float_or_none(
+            xpath_text(item, ns('./jwplayer:duration'), 'duration'))
+
         description = self._html_search_regex(
-            r'<div class="leadtext">\n{0,}?\s{0,}<p>(.*?)</p>',
+            r'(?s)<div class="leadtext">(.+?)</div>',
             webpage, 'description')
 
-        playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
-        playlist = self._download_xml(
-            'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist,
-            video_id)
-
-        namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
-
-        item = playlist.find('tracklist/item')
-        streamer = item.find('{%s}streamer' % namespace).text
+        thumbnail = self._html_search_meta(
+            'thumbnail', webpage, 'thumbnail')
+        if thumbnail:
+            thumbnail = compat_urlparse.urljoin(url, thumbnail)
 
         return {
             'id': video_id,
-            'title': self._html_search_meta('title', webpage),
+            'url': streamer.replace('rtmpt', 'rtmp'),
+            'play_path': 'mp4:%s' % video_file,
+            'ext': 'flv',
+            'title': title,
             'description': description,
-            'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
-            'protocol': 'rtmp',
-            'url': streamer.replace('rtmpt', 'http'),
-            'play_path': 'mp4:' + item.find('{%s}file' % namespace).text,
-            'tc_url': streamer,
-            'ext': 'mp4'
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': duration,
         }

From 63f3cab4aee5da45dc9b91a9661d5d52b5a72ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 21:09:08 +0600
Subject: [PATCH 0197/2145] [rtbf] Fix extraction (Closes #5803)

---
 youtube_dl/extractor/rtbf.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index dce64e151..5a381d9ce 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -1,10 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unescapeHTML,
+)
 
 
 class RTBFIE(InfoExtractor):
@@ -16,25 +17,24 @@ class RTBFIE(InfoExtractor):
             'id': '1921274',
             'ext': 'mp4',
             'title': 'Les Diables au coeur (épisode 2)',
-            'description': 'Football - Diables Rouges',
             'duration': 3099,
-            'timestamp': 1398456336,
-            'upload_date': '20140425',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        page = self._download_webpage('https://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
+        webpage = self._download_webpage(
+            'http://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
 
-        data = json.loads(self._html_search_regex(
-            r'<div class="js-player-embed(?: player-embed)?" data-video="([^"]+)"', page, 'data video'))['data']
+        data = self._parse_json(
+            unescapeHTML(self._search_regex(
+                r'data-video="([^"]+)"', webpage, 'data video')),
+            video_id)
 
         video_url = data.get('downloadUrl') or data.get('url')
 
-        if data['provider'].lower() == 'youtube':
+        if data.get('provider').lower() == 'youtube':
             return self.url_result(video_url, 'Youtube')
 
         return {
@@ -42,8 +42,8 @@ class RTBFIE(InfoExtractor):
             'url': video_url,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
-            'thumbnail': data['thumbnail']['large'],
+            'thumbnail': data.get('thumbnail'),
             'duration': data.get('duration') or data.get('realDuration'),
-            'timestamp': data['created'],
-            'view_count': data['viewCount'],
+            'timestamp': int_or_none(data.get('created')),
+            'view_count': int_or_none(data.get('viewCount')),
         }

From 2ad5708c43a8672da547fa279e71b20c327793d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 21:25:00 +0600
Subject: [PATCH 0198/2145] [arte:future] Switch to `search_regex` for now
 (Closes #5801)

---
 youtube_dl/extractor/arte.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 8273bd6c9..fce38248d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -195,7 +195,9 @@ class ArteTVFutureIE(ArteTVPlus7IE):
     def _real_extract(self, url):
         anchor_id, lang = self._extract_url_info(url)
         webpage = self._download_webpage(url, anchor_id)
-        row = get_element_by_id(anchor_id, webpage)
+        row = self._search_regex(
+            r'(?s)id="%s"[^>]*>.+?(<div[^>]*arte_vp_url[^>]*>)' % anchor_id,
+            webpage, 'row')
         return self._extract_from_webpage(row, anchor_id, lang)
 
 

From 04b3b3df05a26a361441754afeb7ff24d0c1f559 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 11:58:52 +0200
Subject: [PATCH 0199/2145] [youtube] Remove the nondash formats (fixes #5774)

Since we use fixed values for some fields like width and height they can be wrong, and would get picked by some formats filters.
For example for https://www.youtube.com/watch?v=EQCrhbBxsjA the biggest height is 720 and for nondash formats it's set to 1440, so -f 'bestvideo[height>=1200]+bestaudio' would incorrectly pick the nondash format, instead it should report that the requested format is not available.
---
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1f9940cf5..0301682b8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1126,12 +1126,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
                 else:
-                    # Hide the formats we found through non-DASH
+                    # Remove the formats we found through non-DASH, they
+                    # contain less info and it can be wrong, because we use
+                    # fixed values (for example the resolution). See
+                    # https://github.com/rg3/youtube-dl/issues/5774 for an
+                    # example.
                     dash_keys = set(df['format_id'] for df in dash_formats)
-                    for f in formats:
-                        if f['format_id'] in dash_keys:
-                            f['format_id'] = 'nondash-%s' % f['format_id']
-                            f['preference'] = f.get('preference', 0) - 10000
+                    formats = [f for f in formats if f['format_id'] not in dash_keys]
                     formats.extend(dash_formats)
 
         # Check for malformed aspect ratio

From 4b4e1af059c0922da9770a79a68a471277303f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 24 May 2015 18:46:29 +0200
Subject: [PATCH 0200/2145] [arte] Remove unused import

---
 youtube_dl/extractor/arte.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index fce38248d..76de24477 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
     unified_strdate,
-    get_element_by_id,
     get_element_by_attribute,
     int_or_none,
     qualities,

From d41ebe146b693011eb1020ca9cd935e7db892d0b Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 24 May 2015 23:57:47 +0600
Subject: [PATCH 0201/2145] [tenplay] Fix formats and modernize (Closes #5806)

---
 youtube_dl/extractor/tenplay.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index 466155ef8..f6694149b 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -2,6 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
 
 class TenPlayIE(InfoExtractor):
@@ -49,18 +53,23 @@ class TenPlayIE(InfoExtractor):
             if protocol == 'rtmp':
                 url = url.replace('&mp4:', '')
 
+                tbr = int_or_none(rendition.get('encodingRate'), 1000)
+
             formats.append({
-                'format_id': '_'.join(['rtmp', rendition['videoContainer'].lower(), rendition['videoCodec'].lower()]),
-                'width': rendition['frameWidth'],
-                'height': rendition['frameHeight'],
-                'tbr': rendition['encodingRate'] / 1024,
-                'filesize': rendition['size'],
+                'format_id': '_'.join(
+                    ['rtmp', rendition['videoContainer'].lower(),
+                     rendition['videoCodec'].lower(), '%sk' % tbr]),
+                'width': int_or_none(rendition['frameWidth']),
+                'height': int_or_none(rendition['frameHeight']),
+                'tbr': tbr,
+                'filesize': int_or_none(rendition['size']),
                 'protocol': protocol,
                 'ext': ext,
                 'vcodec': rendition['videoCodec'].lower(),
                 'container': rendition['videoContainer'].lower(),
                 'url': url,
             })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
@@ -74,8 +83,8 @@ class TenPlayIE(InfoExtractor):
                 'url': json['thumbnailURL']
             }],
             'thumbnail': json['videoStillURL'],
-            'duration': json['length'] / 1000,
-            'timestamp': float(json['creationDate']) / 1000,
-            'uploader': json['customFields']['production_company_distributor'] if 'production_company_distributor' in json['customFields'] else 'TENplay',
-            'view_count': json['playsTotal']
+            'duration': float_or_none(json.get('length'), 1000),
+            'timestamp': float_or_none(json.get('creationDate'), 1000),
+            'uploader': json.get('customFields', {}).get('production_company_distributor') or 'TENplay',
+            'view_count': int_or_none(json.get('playsTotal')),
         }

From 7198063d96003050eccb0ea59cc938f0388c0606 Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 15:26:59 -0500
Subject: [PATCH 0202/2145] [pinkbike] new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pinkbike.py | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/pinkbike.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..80bec39da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -394,6 +394,7 @@ from .pbs import PBSIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
+from .pinkbike import PinkbikeIE
 from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
new file mode 100644
index 000000000..4a15c1835
--- /dev/null
+++ b/youtube_dl/extractor/pinkbike.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class PinkbikeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pinkbike\.com/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.pinkbike.com/video/402811/',
+        'md5': '4814b8ca7651034cd87e3361d5c2155a',
+        'info_dict': {
+            'id': '402811',
+            'ext': 'mp4',
+            'title': 'Brandon Semenuk - RAW 100',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'location': 'Victoria, British Columbia, Canada',
+            'uploader_id': 'revelco',
+            'upload_date': '20150406',
+            'description': 'Official release: www.redbull.ca/rupertwalker',
+            'duration': '100'
+        }
+    }, {
+        'url': 'http://www.pinkbike.com/video/406629/',
+        'md5': 'c7a3e19a2bd5cde5a1cda6b2b46caa74',
+        'info_dict': {
+            'id': '406629',
+            'ext': 'mp4',
+            'title': 'Chromag: Reece Wallace in Utah',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'location': 'Whistler, British Columbia, Canada',
+            'uploader_id': 'Chromagbikes',
+            'upload_date': '20150505',
+            'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
+            'duration': '180'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        title = title[:-len(' Video - Pinkbike')]
+
+        description = self._html_search_meta('description', webpage, 'description')
+        description = description[len(title + '. '):]
+
+        uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
+
+        upload_date = self._html_search_regex(
+            r'class="fullTime"\s*title="([0-9]{4}(?:-[0-9]{2}){2})"',
+            webpage, 'upload_date')
+        upload_date = upload_date.replace('-', '')
+
+        location = self._html_search_regex(
+            r'<dt>Location</dt>\n?\s*<dd>\n?(.*?)\s*<img',
+            webpage, 'location')
+
+        formats = re.findall(
+            r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
+            webpage)
+
+        formats = [{'url': fmt[1], 'height': fmt[0]} for fmt in formats]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': self._html_search_meta('video:duration', webpage, 'duration'),
+            'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'location': location,
+            'formats': formats
+        }

From 2c935c0c7224a3332ff9f0fd83e8c074cfbe2c9d Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 16:30:03 -0500
Subject: [PATCH 0203/2145] [pinkbike] converted duration to int

---
 youtube_dl/extractor/pinkbike.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 4a15c1835..66605ddbe 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -20,7 +20,7 @@ class PinkbikeIE(InfoExtractor):
             'uploader_id': 'revelco',
             'upload_date': '20150406',
             'description': 'Official release: www.redbull.ca/rupertwalker',
-            'duration': '100'
+            'duration': 100
         }
     }, {
         'url': 'http://www.pinkbike.com/video/406629/',
@@ -34,7 +34,7 @@ class PinkbikeIE(InfoExtractor):
             'uploader_id': 'Chromagbikes',
             'upload_date': '20150505',
             'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
-            'duration': '180'
+            'duration': 180
         }
     }]
 
@@ -69,7 +69,7 @@ class PinkbikeIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': self._html_search_meta('video:duration', webpage, 'duration'),
+            'duration': int(self._html_search_meta('video:duration', webpage, 'duration')),
             'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From 680f9744c4e010ad5111c7711c58c341d5ba24dd Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 16:45:10 -0500
Subject: [PATCH 0204/2145] [pinkbike] used proper conversion methods

---
 youtube_dl/extractor/pinkbike.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 66605ddbe..45c0b1377 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -4,6 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_end,
+    remove_start
+)
 
 
 class PinkbikeIE(InfoExtractor):
@@ -43,10 +48,13 @@ class PinkbikeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        title = title[:-len(' Video - Pinkbike')]
+        title = remove_end(title, ' Video - Pinkbike')
 
         description = self._html_search_meta('description', webpage, 'description')
-        description = description[len(title + '. '):]
+        description = remove_start(description, title + '. ')
+
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration'))
 
         uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
 
@@ -63,13 +71,13 @@ class PinkbikeIE(InfoExtractor):
             r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
             webpage)
 
-        formats = [{'url': fmt[1], 'height': fmt[0]} for fmt in formats]
+        formats = [{'url': fmt[1], 'height': int_or_none(fmt[0])} for fmt in formats]
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': int(self._html_search_meta('video:duration', webpage, 'duration')),
+            'duration': duration,
             'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From b885bae6340b2aa9406501250fdebfbeea54e5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 04:53:53 +0600
Subject: [PATCH 0205/2145] Credit @misterhat for karrierevideos (#5729)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ebed7ebb3..3410e1fb9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -125,3 +125,4 @@ Roman Le Négrate
 Matthias Küch
 Julian Richen
 Ping O.
+Mister Hat

From c6bbdadd79fac001cde15e8fd118b9535427474d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 21:22:13 +0600
Subject: [PATCH 0206/2145] [odnoklassniki] Support extraction from metadata
 URL (Closes #5813)

---
 youtube_dl/extractor/odnoklassniki.py | 33 ++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index fbc521d1a..691f503f5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
     unified_strdate,
     int_or_none,
@@ -11,8 +12,9 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
+        # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
         'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
         'info_dict': {
@@ -20,11 +22,22 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'duration': 100,
-            'upload_date': '20141207',
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
-            'age_limit': 0,
+        },
+    }, {
+        # metadataUrl
+        'url': 'http://ok.ru/video/63567059965189-0',
+        'md5': '9676cf86eff5391d35dea675d224e131',
+        'info_dict': {
+            'id': '63567059965189-0',
+            'ext': 'mp4',
+            'title': 'Девушка без комплексов ...',
+            'duration': 191,
+            'uploader_id': '534380003155',
+            'uploader': 'Андрей Мещанинов',
+            'like_count': int,
         },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
@@ -41,7 +54,15 @@ class OdnoklassnikiIE(InfoExtractor):
                 r'data-attributes="([^"]+)"', webpage, 'player')),
             video_id)
 
-        metadata = self._parse_json(player['flashvars']['metadata'], video_id)
+        flashvars = player['flashvars']
+
+        metadata = flashvars.get('metadata')
+        if metadata:
+            metadata = self._parse_json(metadata, video_id)
+        else:
+            metadata = self._download_json(
+                compat_urllib_parse.unquote(flashvars['metadataUrl']),
+                video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']
         title = movie['title']
@@ -53,11 +74,11 @@ class OdnoklassnikiIE(InfoExtractor):
         uploader = author.get('name')
 
         upload_date = unified_strdate(self._html_search_meta(
-            'ya:ovs:upload_date', webpage, 'upload date'))
+            'ya:ovs:upload_date', webpage, 'upload date', default=None))
 
         age_limit = None
         adult = self._html_search_meta(
-            'ya:ovs:adult', webpage, 'age limit')
+            'ya:ovs:adult', webpage, 'age limit', default=None)
         if adult:
             age_limit = 18 if adult == 'true' else 0
 

From ba2df04b41b62d08e1fd0efaaaf104467133e9a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 21:27:43 +0600
Subject: [PATCH 0207/2145] [odnoklassniki] Make URL explicit

---
 youtube_dl/extractor/odnoklassniki.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 691f503f5..6c7149fe3 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -47,7 +47,8 @@ class OdnoklassnikiIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://ok.ru/video/%s' % video_id, video_id)
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(

From 5d0a33eebcae821ac5d1124043a5ad77a58fa291 Mon Sep 17 00:00:00 2001
From: Alexander Kirk <alexander@kirk.at>
Date: Mon, 25 May 2015 20:12:18 +0200
Subject: [PATCH 0208/2145] rtlnow is now hosted at nowtv.de

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowtv.py    |  90 ++++++++++++++++
 youtube_dl/extractor/rtlnow.py   | 174 -------------------------------
 3 files changed, 91 insertions(+), 175 deletions(-)
 create mode 100644 youtube_dl/extractor/nowtv.py
 delete mode 100644 youtube_dl/extractor/rtlnow.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..bfd07392e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,6 +355,7 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
+from .nowtv import NowTvIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
@@ -438,7 +439,6 @@ from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlNlIE
-from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
new file mode 100644
index 000000000..bf97fe7f4
--- /dev/null
+++ b/youtube_dl/extractor/nowtv.py
@@ -0,0 +1,90 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    qualities,
+    unified_strdate,
+    int_or_none,
+)
+
+class NowTvIE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'''(?x)
+                        (?:https?://)?
+                        (
+                            (?:www\.)?nowtv\.de
+                            /(rtl|rtl2|rtlnitro||superrtl|ntv|vox)(?P<path>/.*?)/player
+                        )'''
+
+    _TESTS = [
+        {
+            'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
+            'info_dict': {
+                'id': '128953',
+                'ext': 'mp4',
+                'title': 'B\u00fcro-Fall \/ Chihuahua \'Joel\'',
+                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
+                'upload_date': '2015-05-23 19:10:00',
+                'duration': '00:51:32',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'Only works from Germany',
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        info_url = 'https://api.nowtv.de/v3/movies' + mobj.group('path') + '?fields=*,format,files,breakpoints,paymentPaytypes,trailers'
+        info = self._download_json(info_url, None)
+
+        video_id = info['id']
+        title = info['title']
+        description = info['articleShort']
+        duration = info['duration']
+        upload_date = unified_strdate(info['broadcastStartDate'])
+        free = info['free']
+        station = info['format']['station']
+        thumbnail = info['format']['defaultImage169Logo']
+
+        if station == 'rtl':
+            base_url = 'http://hls.fra.rtlnow.de/hls-vod-enc/'
+        elif station == 'rtl2':
+            base_url = 'http://hls.fra.rtl2now.de/hls-vod-enc/'
+        elif station == 'vox':
+            base_url = 'http://hls.fra.voxnow.de/hls-vod-enc/'
+        elif station == 'nitro':
+            base_url = 'http://hls.fra.rtlnitronow.de/hls-vod-enc/'
+        elif station == 'ntv':
+            base_url = 'http://hls.fra.n-tvnow.de/hls-vod-enc/'
+        elif station == 'superrtl':
+            base_url = 'http://hls.fra.superrtlnow.de/hls-vod-enc/'
+
+        formats = []
+        for item in info['files']['items']:
+            if item['type'] != 'video/x-abr':
+                continue
+
+            fmt = {
+                'url': base_url + item['path'] + '.m3u8',
+                'tbr': int_or_none(item['bitrate']),
+                'ext': 'mp4',
+                'format_id': int_or_none(item['id']),
+            }
+            formats.append(fmt)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
deleted file mode 100644
index 785a8045e..000000000
--- a/youtube_dl/extractor/rtlnow.py
+++ /dev/null
@@ -1,174 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    unified_strdate,
-    int_or_none,
-)
-
-
-class RTLnowIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'''(?x)
-                        (?:https?://)?
-                        (?P<url>
-                            (?P<domain>
-                                rtl-now\.rtl\.de|
-                                rtl2now\.rtl2\.de|
-                                (?:www\.)?voxnow\.de|
-                                (?:www\.)?rtlnitronow\.de|
-                                (?:www\.)?superrtlnow\.de|
-                                (?:www\.)?n-tvnow\.de)
-                            /+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?
-                            (?:container_id|film_id)=(?P<video_id>[0-9]+)&
-                            player=1(?:&season=[0-9]+)?(?:&.*)?
-                        )'''
-
-    _TESTS = [
-        {
-            'url': 'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
-            'info_dict': {
-                'id': '90419',
-                'ext': 'flv',
-                'title': 'Ahornallee - Folge 1 - Der Einzug',
-                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
-                'upload_date': '20070416',
-                'duration': 1685,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
-        },
-        {
-            'url': 'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
-            'info_dict': {
-                'id': '69756',
-                'ext': 'flv',
-                'title': 'Ärger im Revier - Ein junger Ladendieb, ein handfester Streit u.a.',
-                'description': 'md5:3fb247005ed21a935ffc82b7dfa70cf0',
-                'thumbnail': 'http://autoimg.static-fra.de/rtl2now/219850/1500x1500/image2.jpg',
-                'upload_date': '20120519',
-                'duration': 1245,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
-        },
-        {
-            'url': 'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
-            'info_dict': {
-                'id': '13883',
-                'ext': 'flv',
-                'title': 'Voxtours - Südafrika-Reporter II',
-                'description': 'md5:de7f8d56be6fd4fed10f10f57786db00',
-                'upload_date': '20090627',
-                'duration': 1800,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://superrtlnow.de/medicopter-117/angst.php?film_id=99205&player=1',
-            'info_dict': {
-                'id': '99205',
-                'ext': 'flv',
-                'title': 'Medicopter 117 - Angst!',
-                'description': 're:^Im Therapiezentrum \'Sonnalm\' kommen durch eine Unachtsamkeit die für die B.handlung mit Phobikern gehaltenen Voglespinnen frei\. Eine Ausreißerin',
-                'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
-                'upload_date': '20080928',
-                'duration': 2691,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://rtl-now.rtl.de/der-bachelor/folge-4.php?film_id=188729&player=1&season=5',
-            'info_dict': {
-                'id': '188729',
-                'ext': 'flv',
-                'upload_date': '20150204',
-                'description': 'md5:5e1ce23095e61a79c166d134b683cecc',
-                'title': 'Der Bachelor - Folge 4',
-            }
-        }, {
-            'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
-            'only_matching': True,
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_page_url = 'http://%s/' % mobj.group('domain')
-        video_id = mobj.group('video_id')
-
-        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
-
-        mobj = re.search(r'(?s)<div style="margin-left: 20px; font-size: 13px;">(.*?)<div id="playerteaser">', webpage)
-        if mobj:
-            raise ExtractorError(clean_html(mobj.group(1)), expected=True)
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
-
-        upload_date = unified_strdate(self._html_search_meta('uploadDate', webpage, 'upload date'))
-
-        mobj = re.search(r'<meta itemprop="duration" content="PT(?P<seconds>\d+)S" />', webpage)
-        duration = int(mobj.group('seconds')) if mobj else None
-
-        playerdata_url = self._html_search_regex(
-            r"'playerdata': '(?P<playerdata_url>[^']+)'", webpage, 'playerdata_url')
-
-        playerdata = self._download_xml(playerdata_url, video_id, 'Downloading player data XML')
-
-        videoinfo = playerdata.find('./playlist/videoinfo')
-
-        formats = []
-        for filename in videoinfo.findall('filename'):
-            mobj = re.search(r'(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>.+)', filename.text)
-            if mobj:
-                fmt = {
-                    'url': mobj.group('url'),
-                    'play_path': 'mp4:' + mobj.group('play_path'),
-                    'page_url': video_page_url,
-                    'player_url': video_page_url + 'includes/vodplayer.swf',
-                }
-            else:
-                mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
-                if mobj:
-                    fmt = {
-                        'url': 'rtmpe://fms.rtl.de/' + mobj.group('hoster'),
-                        'play_path': 'mp4:' + mobj.group('play_path'),
-                        'page_url': url,
-                        'player_url': video_page_url + 'includes/vodplayer.swf',
-                    }
-                else:
-                    fmt = {
-                        'url': filename.text,
-                    }
-            fmt.update({
-                'width': int_or_none(filename.get('width')),
-                'height': int_or_none(filename.get('height')),
-                'vbr': int_or_none(filename.get('bitrate')),
-                'ext': 'flv',
-            })
-            formats.append(fmt)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'duration': duration,
-            'formats': formats,
-        }

From 42833b44b5e3810a2875dfb130aefbf5db057c1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:32:43 +0800
Subject: [PATCH 0209/2145] [tf1] Extend _VALID_URL (fixes #5819)

---
 youtube_dl/extractor/tf1.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 025d0877c..656410528 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
-    _TESTS = {
+    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou|www\.tf1)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
             'id': '10635995',
@@ -32,7 +32,10 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From db3ca36403930063b7df3b228a3f297bf278b43c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:37:15 +0800
Subject: [PATCH 0210/2145] [facebook] Move the title extraction warning below
 (fixes #5820)

---
 youtube_dl/extractor/facebook.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 937b28fcc..e8d682716 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -50,7 +50,10 @@ class FacebookIE(InfoExtractor):
             'id': '274175099429670',
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
-        }
+        },
+        'expected_warnings': [
+            'title'
+        ]
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -150,11 +153,11 @@ class FacebookIE(InfoExtractor):
 
         video_title = self._html_search_regex(
             r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
-            fatal=False)
+            default=None)
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
-                webpage, 'alternative title', default=None)
+                webpage, 'alternative title', fatal=False)
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id

From efec4358b9b8ed5726c1f7d7939e8bce49f9100c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:54:41 +0800
Subject: [PATCH 0211/2145] [cinemassacre] Support an alternative form of
 screenwavemedia URL

fixes #5821
---
 youtube_dl/extractor/cinemassacre.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index cf0a7551b..c949a4814 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -60,6 +60,17 @@ class CinemassacreIE(InfoExtractor):
                 'uploader_id': 'Cinemassacre',
                 'title': 'AVGN: McKids',
             }
+        },
+        {
+            'url': 'http://cinemassacre.com/2015/05/25/mario-kart-64-nintendo-64-james-mike-mondays/',
+            'md5': '1376908e49572389e7b06251a53cdd08',
+            'info_dict': {
+                'id': 'Cinemassacre-555779690c440',
+                'ext': 'mp4',
+                'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
+                'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
+                'upload_date': '20150525',
+            }
         }
     ]
 
@@ -72,7 +83,7 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             [
-                r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+                r'src="(http://(?:player2\.screenwavemedia\.com|player\.screenwavemedia\.com/play)/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
                 r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
             webpage, 'player data URL', default=None)

From ff305edd645d6f4307faa9307dea91694a1d217d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:43:00 +0300
Subject: [PATCH 0212/2145] [sockshare] Remove extractor

Haywire since last October.
---
 youtube_dl/extractor/sockshare.py | 83 -------------------------------
 1 file changed, 83 deletions(-)
 delete mode 100644 youtube_dl/extractor/sockshare.py

diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
deleted file mode 100644
index b5fa6f1da..000000000
--- a/youtube_dl/extractor/sockshare.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-)
-
-from .common import InfoExtractor
-
-
-class SockshareIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sockshare\.com/file/(?P<id>[0-9A-Za-z]+)'
-    _FILE_DELETED_REGEX = r'This file doesn\'t exist, or has been removed\.</div>'
-    _TEST = {
-        'url': 'http://www.sockshare.com/file/437BE28B89D799D7',
-        'md5': '9d0bf1cfb6dbeaa8d562f6c97506c5bd',
-        'info_dict': {
-            'id': '437BE28B89D799D7',
-            'title': 'big_buck_bunny_720p_surround.avi',
-            'ext': 'avi',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://sockshare.com/file/%s' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        confirm_hash = self._html_search_regex(r'''(?x)<input\s+
-            type="hidden"\s+
-            value="([^"]*)"\s+
-            name="hash"
-            ''', webpage, 'hash')
-
-        fields = {
-            "hash": confirm_hash.encode('utf-8'),
-            "confirm": "Continue as Free User"
-        }
-
-        post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
-        # Apparently, this header is required for confirmation to work.
-        req.add_header('Host', 'www.sockshare.com')
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        webpage = self._download_webpage(
-            req, video_id, 'Downloading video page')
-
-        video_url = self._html_search_regex(
-            r'<a href="([^"]*)".+class="download_file_link"',
-            webpage, 'file url')
-        video_url = "http://www.sockshare.com" + video_url
-        title = self._html_search_regex((
-            r'<h1>(.+)<strong>',
-            r'var name = "([^"]+)";'),
-            webpage, 'title', default=None)
-        thumbnail = self._html_search_regex(
-            r'<img\s+src="([^"]*)".+?name="bg"',
-            webpage, 'thumbnail', default=None)
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'ext': determine_ext(title),
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 35a4f24a378419d41ea9d9fbcc9790fa457e83d2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:44:46 +0300
Subject: [PATCH 0213/2145] [firedrive] Remove extractor (Closes #3870)

Haywire since last October.
---
 youtube_dl/extractor/firedrive.py | 80 -------------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/firedrive.py

diff --git a/youtube_dl/extractor/firedrive.py b/youtube_dl/extractor/firedrive.py
deleted file mode 100644
index 3191116d9..000000000
--- a/youtube_dl/extractor/firedrive.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-)
-
-
-class FiredriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?firedrive\.com/' + \
-                 '(?:file|embed)/(?P<id>[0-9a-zA-Z]+)'
-    _FILE_DELETED_REGEX = r'<div class="removed_file_image">'
-
-    _TESTS = [{
-        'url': 'https://www.firedrive.com/file/FEB892FA160EBD01',
-        'md5': 'd5d4252f80ebeab4dc2d5ceaed1b7970',
-        'info_dict': {
-            'id': 'FEB892FA160EBD01',
-            'ext': 'flv',
-            'title': 'bbb_theora_486kbit.flv',
-            'thumbnail': 're:^http://.*\.jpg$',
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://firedrive.com/file/%s' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', webpage))
-
-        post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        # Apparently, this header is required for confirmation to work.
-        req.add_header('Host', 'www.firedrive.com')
-
-        webpage = self._download_webpage(req, video_id,
-                                         'Downloading video page')
-
-        title = self._search_regex(r'class="external_title_left">(.+)</div>',
-                                   webpage, 'title')
-        thumbnail = self._search_regex(r'image:\s?"(//[^\"]+)', webpage,
-                                       'thumbnail', fatal=False)
-        if thumbnail is not None:
-            thumbnail = 'http:' + thumbnail
-
-        ext = self._search_regex(r'type:\s?\'([^\']+)\',',
-                                 webpage, 'extension', fatal=False)
-        video_url = self._search_regex(
-            r'file:\s?loadURL\(\'(http[^\']+)\'\),', webpage, 'file url')
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'ext': ext,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 544a8693b7d9321d776987a5104889056955daa2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:53:14 +0300
Subject: [PATCH 0214/2145] Remove Firedrive and Sockshare imports

Oops
---
 youtube_dl/extractor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..80c9cb107 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -149,7 +149,6 @@ from .extremetube import ExtremeTubeIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
-from .firedrive import FiredriveIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
@@ -480,7 +479,6 @@ from .smotri import (
     SmotriBroadcastIE,
 )
 from .snotr import SnotrIE
-from .sockshare import SockshareIE
 from .sohu import SohuIE
 from .soundcloud import (
     SoundcloudIE,

From 7d65242dc3c0c3306b775f0663d325ba55b62379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 May 2015 22:12:26 +0600
Subject: [PATCH 0215/2145] [dailymotion:user] Process user home as user
 (Closes #5823)

---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index db10b8d00..d8f9eb13f 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -225,7 +225,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:old/)?user/(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',
@@ -239,7 +239,8 @@ class DailymotionUserIE(DailymotionPlaylistIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
-        webpage = self._download_webpage(url, user)
+        webpage = self._download_webpage(
+            'https://www.dailymotion.com/user/%s' % user, user)
         full_user = unescapeHTML(self._html_search_regex(
             r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
             webpage, 'user'))

From 5406af92bc3f88b9fe4f26fe20bdaaf0b4968c32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 May 2015 22:16:47 +0600
Subject: [PATCH 0216/2145] [dailymotion:user] Fix _VALID_URL

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index d8f9eb13f..70aa4333c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -225,7 +225,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 7a372b64dfa69d5b2cfd1514b89e8fc0ab7e5874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 01:41:00 +0600
Subject: [PATCH 0217/2145] [pornhub] Do not modify aes key string (Closes
 #5824)

---
 youtube_dl/extractor/pornhub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 0c8b731cf..daa284ea2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -71,7 +71,8 @@ class PornHubIE(InfoExtractor):
 
         video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
+            password = compat_urllib_parse.unquote_plus(
+                self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 
         formats = []

From 1434184c577953ff6fe558ccc6751697791f4076 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 01:42:53 +0600
Subject: [PATCH 0218/2145] [spankwire] Do not modify aes key string

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index b936202f6..06d6e6640 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -71,7 +71,7 @@ class SpankwireIE(InfoExtractor):
             compat_urllib_parse.unquote,
             re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
-            password = self._html_search_regex(
+            password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',
                 webpage, 'password').replace('+', ' ')
             video_urls = list(map(

From b535170b218131afd1165776e611691479627ce8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 04:14:24 +0800
Subject: [PATCH 0219/2145] [bilibili] Skip assertion if HQ videos not
 available

---
 youtube_dl/extractor/bilibili.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 7ca835e31..a8bea2c10 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -80,9 +80,11 @@ class BiliBiliIE(InfoExtractor):
             note='Downloading HQ video info',
             fatal=False,
         )
-        hq_durls = hq_doc.findall('./durl') if hq_doc is not False else itertools.repeat(None)
-
-        assert len(lq_durls) == len(hq_durls)
+        if hq_doc is not False:
+            hq_durls = hq_doc.findall('./durl')
+            assert len(lq_durls) == len(hq_durls)
+        else:
+            hq_durls = itertools.repeat(None)
 
         i = 1
         for lq_durl, hq_durl in zip(lq_durls, hq_durls):

From 6d00a2dcd110f12a0aa110f5479df76613792fbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 04:23:21 +0800
Subject: [PATCH 0220/2145] [bilibili] Catch API call failures

JSON are returned in a failed API call
---
 youtube_dl/extractor/bilibili.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index a8bea2c10..2103ed73a 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 
 import re
 import itertools
+import json
+import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
@@ -67,11 +69,19 @@ class BiliBiliIE(InfoExtractor):
 
         entries = []
 
-        lq_doc = self._download_xml(
+        lq_page = self._download_webpage(
             'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
             video_id,
             note='Downloading LQ video info'
         )
+        try:
+            err_info = json.loads(lq_page)
+            raise ExtractorError(
+                'BiliBili said: ' + err_info['error_text'], expected=True)
+        except ValueError:
+            pass
+
+        lq_doc = ET.fromstring(lq_page)
         lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(

From c23848b3c5244f8ef1501adfd04a32111b12d7ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:20:29 +0800
Subject: [PATCH 0221/2145] [naver] Enhanced error detection

---
 youtube_dl/extractor/naver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index c10405f04..fdd825784 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -35,7 +35,7 @@ class NaverIE(InfoExtractor):
                          webpage)
         if m_id is None:
             m_error = re.search(
-                r'(?s)<div class="nation_error">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                r'(?s)<div class="(?:nation_error|nation_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
                 webpage)
             if m_error:
                 raise ExtractorError(clean_html(m_error.group('msg')), expected=True)

From f8d5e1cfb5d9a8c946a966452d9b86c45182a952 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:44:08 +0800
Subject: [PATCH 0222/2145] [naver] Fix video url (fixes #5809)

RTMP urls in test:naver does not work. Need more investigation.
---
 youtube_dl/extractor/naver.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index fdd825784..925967753 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -16,7 +17,7 @@ from ..utils import (
 class NaverIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tvcast.naver.com/v/81652',
         'info_dict': {
             'id': '81652',
@@ -25,7 +26,18 @@ class NaverIE(InfoExtractor):
             'description': '합격불변의 법칙 메가스터디 | 메가스터디 수학 김상희 선생님이 9월 모의고사 수학A형 16번에서 20번까지 해설강의를 공개합니다.',
             'upload_date': '20130903',
         },
-    }
+    }, {
+        'url': 'http://tvcast.naver.com/v/395837',
+        'md5': '638ed4c12012c458fefcddfd01f173cd',
+        'info_dict': {
+            'id': '395837',
+            'ext': 'mp4',
+            'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
+            'description': 'md5:5bf200dcbf4b66eb1b350d1eb9c753f7',
+            'upload_date': '20150519',
+        },
+        'skip': 'Georestricted',
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -58,14 +70,18 @@ class NaverIE(InfoExtractor):
         formats = []
         for format_el in urls.findall('EncodingOptions/EncodingOption'):
             domain = format_el.find('Domain').text
+            uri = format_el.find('uri').text
             f = {
-                'url': domain + format_el.find('uri').text,
+                'url': compat_urlparse.urljoin(domain, uri),
                 'ext': 'mp4',
                 'width': int(format_el.find('width').text),
                 'height': int(format_el.find('height').text),
             }
             if domain.startswith('rtmp'):
+                # urlparse does not support custom schemes
+                # https://bugs.python.org/issue18828
                 f.update({
+                    'url': domain + uri,
                     'ext': 'flv',
                     'rtmp_protocol': '1',  # rtmpt
                 })

From f9f3e3df9a9fff1b00c7184234c4f607ea3cec81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:51:18 +0800
Subject: [PATCH 0223/2145] [teamcoco] Use determine_ext to determine the video
 type

Some videos does not contain a 'type' field (#5798)
---
 youtube_dl/extractor/teamcoco.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 56be52638..b2a4b1fc0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     qualities,
+    determine_ext,
 )
 from ..compat import compat_ord
 
@@ -108,7 +109,7 @@ class TeamcocoIE(InfoExtractor):
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
-            if filed['type'] == 'hls':
+            if determine_ext(filed['url']) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     filed['url'], video_id, ext='mp4'))
             else:

From f0bfaa2d7d9563975f1f6effa75d28dcdb1c23ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 15:23:34 +0800
Subject: [PATCH 0224/2145] [nrk] Update subtitles test

Subtitle conversion routine is removed, so the subtitles are TTML now. See
1c7e2e64f6328024711d5fa999d4498396f4cb5c
---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 891ee620b..c4e3adb67 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -266,7 +266,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['no']))
-        self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
+        self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
 
 
 class TestRaiSubtitles(BaseTestSubtitles):

From bf24c3d01798fad0a8344a642eb5d46231fd78c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 21:25:07 +0600
Subject: [PATCH 0225/2145] [facebook] Improve title regex (Closes #5816)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index e8d682716..82dc27bc6 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -152,7 +152,7 @@ class FacebookIE(InfoExtractor):
             raise ExtractorError('Cannot find video formats')
 
         video_title = self._html_search_regex(
-            r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)
         if not video_title:
             video_title = self._html_search_regex(

From d90b3854ca9e8602f440cc9439e1cba240192286 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 00:37:00 +0800
Subject: [PATCH 0226/2145] [porn91] Add new extractor for 91porn.com

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/porn91.py   | 62 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/porn91.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 80c9cb107..d20ad286d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -400,6 +400,7 @@ from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
+from .porn91 import Porn91IE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
new file mode 100644
index 000000000..af06af2b7
--- /dev/null
+++ b/youtube_dl/extractor/porn91.py
@@ -0,0 +1,62 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from ..compat import compat_urllib_parse
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class Porn91IE(InfoExtractor):
+    IE_NAME = '91porn'
+    _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
+
+    _TEST = {
+            'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
+            'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+            'info_dict': {
+                'id': '7e42283b4f5ab36da134',
+                'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+                'ext': 'mp4'
+            }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
+        self._set_cookie('91porn.com', 'language', 'cn_CN')
+        webpage = self._download_webpage(url, video_id, "get HTML content")
+        title = re.search(
+            r'<div id="viewvideo-title">(.+?)</div>',
+            webpage,
+            re.DOTALL)
+        assert title
+        title = title.group(1).replace('\n', '')
+
+        # get real url
+        n1 = re.search(r'so.addVariable\(\'file\',\'(\d+)\'', webpage)
+        n2 = re.search(r'so.addVariable\(\'seccode\',\'(.+?)\'', webpage)
+        n3 = re.search(r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage)
+        if not (n1 and n2 and n3):
+            raise ExtractorError("You are Blocked by Server.")
+
+        url_params = compat_urllib_parse.urlencode({
+            'VID': n1.group(1),
+            'mp4': '1',
+            'seccode': n2.group(1),
+            'max_vid': n3.group(1),
+        })
+        t_url = 'http://91porn.com/getfile.php?' + url_params
+        info_cn = self._download_webpage(t_url, video_id, "get real video_url")
+        video_url = re.search(r'file=(http.+?)&', info_cn).group(1)
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+        }
+
+        return info

From b25b645d5106e5b2bf33c640813fe744b63f4730 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 23:20:32 +0600
Subject: [PATCH 0227/2145] [nowtv] Improve and simplify

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowtv.py    | 197 ++++++++++++++++++++++---------
 2 files changed, 139 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bfd07392e..17248ccea 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,7 +355,7 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
-from .nowtv import NowTvIE
+from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index bf97fe7f4..5c91acec6 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,90 +1,169 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    qualities,
-    unified_strdate,
     int_or_none,
+    parse_iso8601,
+    parse_duration,
+    remove_start,
 )
 
-class NowTvIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'''(?x)
-                        (?:https?://)?
-                        (
-                            (?:www\.)?nowtv\.de
-                            /(rtl|rtl2|rtlnitro||superrtl|ntv|vox)(?P<path>/.*?)/player
-                        )'''
 
-    _TESTS = [
-        {
-            'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
-            'info_dict': {
-                'id': '128953',
-                'ext': 'mp4',
-                'title': 'B\u00fcro-Fall \/ Chihuahua \'Joel\'',
-                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
-                'upload_date': '2015-05-23 19:10:00',
-                'duration': '00:51:32',
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
+class NowTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?P<station>rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+
+    _TESTS = [{
+        # rtl
+        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/player',
+        'info_dict': {
+            'id': '203519',
+            'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
+            'ext': 'mp4',
+            'title': 'Die neuen Bauern und eine Hochzeit',
+            'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432580700,
+            'upload_date': '20150525',
+            'duration': 2786,
         },
-    ]
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # rtl2
+        'url': 'http://www.nowtv.de/rtl2/berlin-tag-nacht/berlin-tag-nacht-folge-934/player',
+        'info_dict': {
+            'id': '203481',
+            'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
+            'ext': 'mp4',
+            'title': 'Berlin - Tag & Nacht (Folge 934)',
+            'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432666800,
+            'upload_date': '20150526',
+            'duration': 2641,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # superrtl
+        'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',
+        'info_dict': {
+            'id': '99205',
+            'display_id': 'medicopter-117/angst',
+            'ext': 'mp4',
+            'title': 'Angst!',
+            'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1222632900,
+            'upload_date': '20080928',
+            'duration': 3025,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # ntv
+        'url': 'http://www.nowtv.de/ntv/ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch/player',
+        'info_dict': {
+            'id': '203521',
+            'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
+            'ext': 'mp4',
+            'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
+            'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432751700,
+            'upload_date': '20150527',
+            'duration': 1083,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # vox
+        'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
+        'info_dict': {
+            'id': '128953',
+            'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
+            'ext': 'mp4',
+            'title': "Büro-Fall / Chihuahua 'Joel'",
+            'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432408200,
+            'upload_date': '20150523',
+            'duration': 3092,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        info_url = 'https://api.nowtv.de/v3/movies' + mobj.group('path') + '?fields=*,format,files,breakpoints,paymentPaytypes,trailers'
-        info = self._download_json(info_url, None)
+        display_id = mobj.group('id')
+        station = mobj.group('station')
 
-        video_id = info['id']
-        title = info['title']
-        description = info['articleShort']
-        duration = info['duration']
-        upload_date = unified_strdate(info['broadcastStartDate'])
-        free = info['free']
-        station = info['format']['station']
-        thumbnail = info['format']['defaultImage169Logo']
+        info = self._download_json(
+            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files,breakpoints,paymentPaytypes,trailers,pictures' % display_id,
+            display_id)
 
-        if station == 'rtl':
-            base_url = 'http://hls.fra.rtlnow.de/hls-vod-enc/'
-        elif station == 'rtl2':
-            base_url = 'http://hls.fra.rtl2now.de/hls-vod-enc/'
-        elif station == 'vox':
-            base_url = 'http://hls.fra.voxnow.de/hls-vod-enc/'
-        elif station == 'nitro':
-            base_url = 'http://hls.fra.rtlnitronow.de/hls-vod-enc/'
-        elif station == 'ntv':
-            base_url = 'http://hls.fra.n-tvnow.de/hls-vod-enc/'
-        elif station == 'superrtl':
-            base_url = 'http://hls.fra.superrtlnow.de/hls-vod-enc/'
+        video_id = compat_str(info['id'])
+
+        if info.get('geoblocked'):
+            raise ExtractorError(
+                'Video %s is not available from your location due to geo restriction' % video_id,
+                expected=True)
+
+        f = info.get('format', {})
+        station = f.get('station') or station
+
+        STATIONS = {
+            'rtl': 'rtlnow',
+            'rtl2': 'rtl2now',
+            'vox': 'voxnow',
+            'nitro': 'rtlnitronow',
+            'ntv': 'n-tvnow',
+            'superrtl': 'superrtlnow'
+        }
 
         formats = []
         for item in info['files']['items']:
-            if item['type'] != 'video/x-abr':
-                continue
-
-            fmt = {
-                'url': base_url + item['path'] + '.m3u8',
-                'tbr': int_or_none(item['bitrate']),
+            item_path = remove_start(item['path'], '/')
+            tbr = int_or_none(item['bitrate'])
+            m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)
+            m3u8_url = m3u8_url.replace('now/', 'now/videos/')
+            formats.append({
+                'url': m3u8_url,
+                'format_id': '%s-%sk' % (item['id'], tbr),
                 'ext': 'mp4',
-                'format_id': int_or_none(item['id']),
-            }
-            formats.append(fmt)
+                'tbr': tbr,
+            })
         self._sort_formats(formats)
 
+        title = info['title']
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
         }

From 703d78bbf5edf73f60447ac273c0d303d28cc340 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 01:37:24 +0800
Subject: [PATCH 0228/2145] [porn91] change re to _search_regex

---
 youtube_dl/extractor/porn91.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index af06af2b7..f3a97df64 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
@@ -29,30 +28,32 @@ class Porn91IE(InfoExtractor):
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
-        title = re.search(
-            r'<div id="viewvideo-title">(.+?)</div>',
-            webpage,
-            re.DOTALL)
+        title = self._search_regex(
+            r'<div id="viewvideo-title">(?P<title>.+?)</div>',
+            webpage, 'title', flags=re.DOTALL)
         assert title
-        title = title.group(1).replace('\n', '')
+        title = title.replace('\n', '')
 
         # get real url
-        n1 = re.search(r'so.addVariable\(\'file\',\'(\d+)\'', webpage)
-        n2 = re.search(r'so.addVariable\(\'seccode\',\'(.+?)\'', webpage)
-        n3 = re.search(r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage)
+        n1 = self._search_regex(
+            r'so.addVariable\(\'file\',\'(?P<n1>\d+)\'', webpage, 'n1')
+        n2 = self._search_regex(
+            r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
+        n3 = self._search_regex(
+            r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
         if not (n1 and n2 and n3):
             raise ExtractorError("You are Blocked by Server.")
-
         url_params = compat_urllib_parse.urlencode({
-            'VID': n1.group(1),
+            'VID': n1,
             'mp4': '1',
-            'seccode': n2.group(1),
-            'max_vid': n3.group(1),
+            'seccode': n2,
+            'max_vid': n3,
         })
         t_url = 'http://91porn.com/getfile.php?' + url_params
         info_cn = self._download_webpage(t_url, video_id, "get real video_url")
-        video_url = re.search(r'file=(http.+?)&', info_cn).group(1)
+        video_url = self._search_regex(r'file=(?P<url>http.+?)&', info_cn, 'url')
 
+        # construct info
         info = {
             'id': video_id,
             'title': title,

From 9b254aa177d58b7b4c4f44dce8c38fa7978c7df6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 23:41:43 +0600
Subject: [PATCH 0229/2145] [nowtv] Add non-free video check

---
 youtube_dl/extractor/nowtv.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 5c91acec6..295168432 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -120,10 +120,15 @@ class NowTVIE(InfoExtractor):
 
         video_id = compat_str(info['id'])
 
-        if info.get('geoblocked'):
-            raise ExtractorError(
-                'Video %s is not available from your location due to geo restriction' % video_id,
-                expected=True)
+        files = info['files']
+        if not files:
+            if info.get('geoblocked', False):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
 
         f = info.get('format', {})
         station = f.get('station') or station
@@ -138,7 +143,7 @@ class NowTVIE(InfoExtractor):
         }
 
         formats = []
-        for item in info['files']['items']:
+        for item in files['items']:
             item_path = remove_start(item['path'], '/')
             tbr = int_or_none(item['bitrate'])
             m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)

From ff4a1279f2d40fdba3287d4e7949bd8caa89eb04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 May 2015 01:15:04 +0600
Subject: [PATCH 0230/2145] [nowtv] Do not request unnecessary metadata

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 295168432..d39bbde99 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -115,7 +115,7 @@ class NowTVIE(InfoExtractor):
         station = mobj.group('station')
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files,breakpoints,paymentPaytypes,trailers,pictures' % display_id,
+            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files' % display_id,
             display_id)
 
         video_id = compat_str(info['id'])

From 9e0b5791281c68e5773555688928184064396011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 May 2015 01:26:14 +0600
Subject: [PATCH 0231/2145] [nowtv] Add test for rtlnitro

---
 youtube_dl/extractor/nowtv.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index d39bbde99..173e46cd8 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -53,6 +53,24 @@ class NowTVIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        # rtlnitro
+        'url': 'http://www.nowtv.de/rtlnitro/alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00/player',
+        'info_dict': {
+            'id': '165780',
+            'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
+            'ext': 'mp4',
+            'title': 'Hals- und Beinbruch',
+            'description': 'md5:b50d248efffe244e6f56737f0911ca57',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432415400,
+            'upload_date': '20150523',
+            'duration': 2742,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # superrtl
         'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',

From f9355dc989362f31d8e21ccb8fa765546a2360f2 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 17:00:09 +0800
Subject: [PATCH 0232/2145] [youku] update youku

---
 youtube_dl/extractor/youku.py | 248 ++++++++++++++++++++++------------
 1 file changed, 162 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 97b98bbe8..8d86c3f45 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,123 +1,199 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-import math
-import random
 import re
-import time
+import base64
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
-
+from ..utils import ExtractorError
 
 class YoukuIE(InfoExtractor):
+    IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
         (?:
             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
             youku:)
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
+
     _TEST = {
-        'url': 'http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html',
-        'md5': 'ffe3f2e435663dc2d1eea34faeff5b5b',
-        'params': {
-            'test': False
-        },
-        'info_dict': {
-            'id': 'XNDgyMDQ2NTQw_part00',
-            'ext': 'flv',
-            'title': 'youtube-dl test video "\'/\\ä↭𝕐'
-        }
+            'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
+            'md5': '5f3af4192eabacc4501508d54a8cabd7',
+            'info_dict': {
+                'id': 'XMTc1ODE5Njcy',
+                'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
+                'ext': 'flv'
+            }
     }
 
-    def _gen_sid(self):
-        nowTime = int(time.time() * 1000)
-        random1 = random.randint(1000, 1998)
-        random2 = random.randint(1000, 9999)
+    def construct_video_urls(self, data1, data2):
+        # get sid, token
+        def yk_t(s1, s2):
+            ls = list(range(256))
+            t = 0
+            for i in range(256):
+                t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
+                ls[i], ls[t] = ls[t], ls[i]
+            s, x, y = '', 0, 0
+            for i in range(len(s2)):
+                y = (y + 1) % 256
+                x = (x + ls[y]) % 256
+                ls[x], ls[y] = ls[y], ls[x]
+                s += chr((s2[i] ^ ls[(ls[x]+ls[y]) % 256]))
+            return s
 
-        return "%d%d%d" % (nowTime, random1, random2)
+        sid, token = yk_t(
+            'becaf9be', base64.b64decode(bytes(data2['ep'], 'ascii'))
+        ).split('_')
 
-    def _get_file_ID_mix_string(self, seed):
-        mixed = []
-        source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
-        seed = float(seed)
-        for i in range(len(source)):
-            seed = (seed * 211 + 30031) % 65536
-            index = math.floor(seed / 65536 * len(source))
-            mixed.append(source[int(index)])
-            source.remove(source[int(index)])
-        # return ''.join(mixed)
-        return mixed
+        # get oip
+        oip = data2['ip']
 
-    def _get_file_id(self, fileId, seed):
-        mixed = self._get_file_ID_mix_string(seed)
-        ids = fileId.split('*')
-        realId = []
-        for ch in ids:
-            if ch:
-                realId.append(mixed[int(ch)])
-        return ''.join(realId)
+        # get fileid
+        string_ls = list(
+            'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
+        shuffled_string_ls = []
+        seed = data1['seed']
+        N = len(string_ls)
+        for ii in range(N):
+            seed = (seed * 0xd3 + 0x754f) % 0x10000
+            idx = seed * len(string_ls) // 0x10000
+            shuffled_string_ls.append(string_ls[idx])
+            del string_ls[idx]
+
+        fileid_dict = {}
+        for format in data1['streamtypes']:
+            streamfileid = [
+                int(i) for i in data1['streamfileids'][format].strip('*').split('*')]
+            fileid = ''.join(
+                [shuffled_string_ls[i] for i in streamfileid])
+            fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
+
+        def get_fileid(format, n):
+            fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            return fileid
+
+        # get ep
+        def generate_ep(format, n):
+            fileid = get_fileid(format, n)
+            ep_t = yk_t(
+                'bf7e5f01',
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii'))
+            ep = base64.b64encode(bytes(ep_t, 'latin')).decode()
+            ep = ep.replace('+', '%2B')
+            ep = ep.replace('/', '%2F')
+            ep = ep.replace('=', '%2D')
+            return ep
+
+        # generate video_urls
+        video_urls_dict = {}
+        for format in data1['streamtypes']:
+            video_urls = []
+            for dt in data1['segs'][format]:
+                n = str(int(dt['no']))
+                video_url = \
+                    'http://k.youku.com/player/getFlvPath/' + \
+                    'sid/' + sid + \
+                    '_' + str(int(n)+1).zfill(2) + \
+                    '/st/' + self.parse_ext_l(format) + \
+                    '/fileid/' + get_fileid(format, n)  + '?' + \
+                    'K=' + str(dt['k']) + \
+                    '&hd=' + self.get_hd(format) + \
+                    '&myp=0' + \
+                    '&ts=' + str(dt['seconds']) + \
+                    '&ypp=0&ctype=12&ev=1' + \
+                    '&token=' + str(token) + \
+                    '&oip=' + str(oip) + \
+                    '&ep=' + generate_ep(format, n)
+                video_urls.append(video_url)
+            video_urls_dict[format] = video_urls
+
+        return video_urls_dict
+
+    def get_hd(self, fm):
+        hd_id_dict = {
+            'flv': '0',
+            'mp4': '1',
+            'hd2': '2',
+            'hd3': '3',
+            '3gp': '0',
+            '3gphd': '1'
+        }
+        return hd_id_dict[fm]
+
+    def parse_ext_l(self, fm):
+        ext_dict = {
+            'flv': 'flv',
+            'mp4': 'mp4',
+            'hd2': 'flv',
+            'hd3': 'flv',
+            '3gp': 'flv',
+            '3gphd': 'mp4',
+        }
+        return ext_dict[fm]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
+        # request basic data
+        data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        data2_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id
 
-        config = self._download_json(info_url, video_id)
+        raw_data1 = self._download_json(data1_url, video_id)
+        raw_data2 = self._download_json(data2_url, video_id)
+        data1 = raw_data1['data'][0]
+        data2 = raw_data2['data'][0]
 
-        error_code = config['data'][0].get('error_code')
+        error_code = data1.get('error_code')
         if error_code:
             # -8 means blocked outside China.
-            error = config['data'][0].get('error')  # Chinese and English, separated by newline.
-            raise ExtractorError(error or 'Server reported error %i' % error_code,
-                                 expected=True)
+            # Chinese and English, separated by newline.
+            error = data1.get('error')
+            raise ExtractorError(
+                error or 'Server reported error %i' %
+                error_code,
+                expected=True)
 
-        video_title = config['data'][0]['title']
-        seed = config['data'][0]['seed']
+        title = data1['title']
 
-        format = self._downloader.params.get('format', None)
-        supported_format = list(config['data'][0]['streamfileids'].keys())
+        # generate video_urls_dict
+        video_urls_dict = self.construct_video_urls(data1, data2)
 
-        # TODO proper format selection
-        if format is None or format == 'best':
-            if 'hd2' in supported_format:
-                format = 'hd2'
-            else:
-                format = 'flv'
-            ext = 'flv'
-        elif format == 'worst':
-            format = 'mp4'
-            ext = 'mp4'
-        else:
-            format = 'flv'
-            ext = 'flv'
+        # construct info
+        entries = []
+        for fm in data1['streamtypes']:
+            #formats = []
+            video_urls = video_urls_dict[fm]
+            for i in range(len(video_urls)):
+                if len(entries) < i+1:
+                    entries.append({'formats': []})
+                entries[i]['formats'].append(
+                    {
+                        'url': video_urls[i],
+                        'format_id': fm,
+                        'ext': self.parse_ext_l(fm),
+                        'filesize': int(data1['segs'][fm][i]['size'])
+                    }
+                )
 
-        fileid = config['data'][0]['streamfileids'][format]
-        keys = [s['k'] for s in config['data'][0]['segs'][format]]
-        # segs is usually a dictionary, but an empty *list* if an error occured.
-
-        files_info = []
-        sid = self._gen_sid()
-        fileid = self._get_file_id(fileid, seed)
-
-        # column 8,9 of fileid represent the segment number
-        # fileid[7:9] should be changed
-        for index, key in enumerate(keys):
-            temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
-            download_url = 'http://k.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
+        for i in range(len(entries)):
+            entries[i].update(
+                {
+                    'id': '_part%d' % (i+1),
+                    'title': title,
+                }
+            )
 
+        if len(entries) > 1:
             info = {
-                'id': '%s_part%02d' % (video_id, index),
-                'url': download_url,
-                'uploader': None,
-                'upload_date': None,
-                'title': video_title,
-                'ext': ext,
+                '_type': 'multi_video',
+                'id': video_id,
+                'title': title,
+                'entries': entries,
             }
-            files_info.append(info)
+        else:
+            info = entries[0]
+            info['id'] = video_id
 
-        return files_info
+        return info

From ca45246627f5a67a7c82cd40a11e5c4ff5f68871 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 21:04:58 +0800
Subject: [PATCH 0233/2145] [youku] compatible for python > 3.3 or > 2.7

---
 youtube_dl/extractor/youku.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 8d86c3f45..7a07c8a5f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import sys
+pyvs = sys.version_info[0]
 import re
 import base64
 
@@ -34,16 +36,23 @@ class YoukuIE(InfoExtractor):
             for i in range(256):
                 t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s, x, y = '', 0, 0
+            s = '' if pyvs == 3 else b''
+            x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
-                s += chr((s2[i] ^ ls[(ls[x]+ls[y]) % 256]))
+                if isinstance(s2[i], int):
+                    s += chr(s2[i] ^ ls[(ls[x]+ls[y]) % 256])
+                else:
+                    s += chr(ord(s2[i]) ^ ls[(ls[x]+ls[y]) % 256])
             return s
 
         sid, token = yk_t(
-            'becaf9be', base64.b64decode(bytes(data2['ep'], 'ascii'))
+            'becaf9be',
+            base64.b64decode(bytes(data2['ep'], 'ascii')) \
+                if pyvs == 3 \
+                else base64.b64decode(data2['ep'])
         ).split('_')
 
         # get oip
@@ -78,8 +87,15 @@ class YoukuIE(InfoExtractor):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
                 'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii'))
-            ep = base64.b64encode(bytes(ep_t, 'latin')).decode()
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
+                if pyvs == 3 \
+                else ('%s_%s_%s' % (sid, fileid, token))
+            )
+            ep = base64.b64encode(
+                bytes(ep_t, 'latin') \
+                if pyvs == 3 \
+                else ep_t
+            ).decode()
             ep = ep.replace('+', '%2B')
             ep = ep.replace('/', '%2F')
             ep = ep.replace('=', '%2D')

From 806598b94dec1268566ae71d671116060f7971d6 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 08:21:24 +0800
Subject: [PATCH 0234/2145] [porn91] the one that _search_regex returns not
 needs to be checked

---
 youtube_dl/extractor/porn91.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index f3a97df64..b62eec92d 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -31,7 +31,6 @@ class Porn91IE(InfoExtractor):
         title = self._search_regex(
             r'<div id="viewvideo-title">(?P<title>.+?)</div>',
             webpage, 'title', flags=re.DOTALL)
-        assert title
         title = title.replace('\n', '')
 
         # get real url
@@ -41,8 +40,6 @@ class Porn91IE(InfoExtractor):
             r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
         n3 = self._search_regex(
             r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
-        if not (n1 and n2 and n3):
-            raise ExtractorError("You are Blocked by Server.")
         url_params = compat_urllib_parse.urlencode({
             'VID': n1,
             'mp4': '1',

From 1498940b10a3f43490c05045ebe7a517267a2bff Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 10:13:09 +0800
Subject: [PATCH 0235/2145] [youku] compare bytes and str for compatible; use
 compat_urllib_parse for making video_url

---
 youtube_dl/extractor/youku.py | 38 +++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 7a07c8a5f..063f2e10e 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,14 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import sys
-pyvs = sys.version_info[0]
 import re
 import base64
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
+from ..compat import compat_urllib_parse
+
+bytes_is_str = (bytes == str)  # for compatible
+
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
@@ -36,7 +38,7 @@ class YoukuIE(InfoExtractor):
             for i in range(256):
                 t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s = '' if pyvs == 3 else b''
+            s = '' if not bytes_is_str else b''
             x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
@@ -51,7 +53,7 @@ class YoukuIE(InfoExtractor):
         sid, token = yk_t(
             'becaf9be',
             base64.b64decode(bytes(data2['ep'], 'ascii')) \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else base64.b64decode(data2['ep'])
         ).split('_')
 
@@ -88,17 +90,14 @@ class YoukuIE(InfoExtractor):
             ep_t = yk_t(
                 'bf7e5f01',
                 bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else ('%s_%s_%s' % (sid, fileid, token))
             )
             ep = base64.b64encode(
                 bytes(ep_t, 'latin') \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else ep_t
             ).decode()
-            ep = ep.replace('+', '%2B')
-            ep = ep.replace('/', '%2F')
-            ep = ep.replace('=', '%2D')
             return ep
 
         # generate video_urls
@@ -107,20 +106,25 @@ class YoukuIE(InfoExtractor):
             video_urls = []
             for dt in data1['segs'][format]:
                 n = str(int(dt['no']))
+                param = {
+                    'K': dt['k'],
+                    'hd': self.get_hd(format),
+                    'myp': 0,
+                    'ts': dt['seconds'],
+                    'ypp': 0,
+                    'ctype': 12,
+                    'ev': 1,
+                    'token': token,
+                    'oip': oip,
+                    'ep': generate_ep(format, n)
+                }
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
                     '_' + str(int(n)+1).zfill(2) + \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n)  + '?' + \
-                    'K=' + str(dt['k']) + \
-                    '&hd=' + self.get_hd(format) + \
-                    '&myp=0' + \
-                    '&ts=' + str(dt['seconds']) + \
-                    '&ypp=0&ctype=12&ev=1' + \
-                    '&token=' + str(token) + \
-                    '&oip=' + str(oip) + \
-                    '&ep=' + generate_ep(format, n)
+                    compat_urllib_parse.urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
 

From 84e1e036c2cb7311cdea14763bec3322403a8d54 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 May 2015 12:44:31 +0800
Subject: [PATCH 0236/2145] [senate] Extend _VALID_URL (fixes #5836)

---
 youtube_dl/extractor/senateisvp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index d3b8a1be4..9c53704ea 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -48,7 +48,7 @@ class SenateISVPIE(InfoExtractor):
         ["arch", "", "http://ussenate-f.akamaihd.net/"]
     ]
     _IE_NAME = 'senate.gov'
-    _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
+    _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
@@ -72,12 +72,16 @@ class SenateISVPIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Integrated Senate Video Player'
         }
+    }, {
+        # From http://www.c-span.org/video/?96791-1
+        'url': 'http://www.senate.gov/isvp?type=live&comm=banking&filename=banking012715',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _search_iframe_url(webpage):
         mobj = re.search(
-            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/\?[^'\"]+)['\"]",
+            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/?\?[^'\"]+)['\"]",
             webpage)
         if mobj:
             return mobj.group('url')

From eb6cb9fbe934fe99a35af22065cf91063d416c12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 29 May 2015 07:52:17 +0200
Subject: [PATCH 0237/2145] release 2015.05.29

---
 README.md              | 4 ++--
 docs/supportedsites.md | 9 ++++-----
 youtube_dl/version.py  | 2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index e51bb5343..f3d83c89f 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress                    Do not print progress bar
     --console-title                  Display progress in console titlebar
     -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages to debug problems (very verbose)
+    --dump-pages                     Print downloaded pages encoded using base64 to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging
@@ -220,7 +220,7 @@ which means you can modify it, redistribute it or use it however you like.
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
-                                     parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
+                                     parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a4879bd9a..a421ae62b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -26,8 +26,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
- - **AppleDailyAnimationNews**
- - **AppleDailyRealtimeNews**
+ - **AppleDaily**
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -152,7 +151,6 @@
  - **fc2**
  - **fernsehkritik.tv**
  - **fernsehkritik.tv:postecke**
- - **Firedrive**
  - **Firstpost**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
@@ -230,6 +228,7 @@
  - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
  - **Karaoketv**
+ - **KarriereVideos**
  - **keek**
  - **KeezMovies**
  - **KhanAcademy**
@@ -322,6 +321,7 @@
  - **NosVideo**
  - **novamov**: NovaMov
  - **Nowness**
+ - **NowTV**
  - **nowvideo**: NowVideo
  - **npo.nl**
  - **npo.nl:live**
@@ -393,7 +393,6 @@
  - **Rte**
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
- - **RTLnow**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
@@ -431,7 +430,6 @@
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
- - **Sockshare**
  - **Sohu**
  - **soundcloud**
  - **soundcloud:playlist**
@@ -564,6 +562,7 @@
  - **vier:videos**
  - **Viewster**
  - **viki**
+ - **viki:channel**
  - **vimeo**
  - **vimeo:album**
  - **vimeo:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b33385153..653710131 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.20'
+__version__ = '2015.05.29'

From d6aa68ce75ef4f4d27fbf7103edfda1f92ba70b2 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 29 May 2015 12:47:20 +0200
Subject: [PATCH 0238/2145] [postprocessor/embedthumbnail] embed mp4 too (fixes
 #5840)

---
 youtube_dl/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 8f825f785..774494efd 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -49,7 +49,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        elif info['ext'] == 'm4a':
+        elif info['ext'] in ['m4a', 'mp4']:
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
@@ -82,6 +82,6 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 os.remove(encodeFilename(filename))
                 os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         else:
-            raise EmbedThumbnailPPError('Only mp3 and m4a are supported for thumbnail embedding for now.')
+            raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
 
         return [], info

From 605ec701b7b4cd120a9acb33bfcc4306719b59b4 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 23:32:04 +0800
Subject: [PATCH 0239/2145] [iqiyi] Add new extractor for iqiyi.com

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/iqiyi.py    | 214 +++++++++++++++++++++++++++++++
 2 files changed, 215 insertions(+)
 create mode 100644 youtube_dl/extractor/iqiyi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 80c9cb107..85c1b1a3a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -229,6 +229,7 @@ from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
+from .iqiyi import IqiyiIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
new file mode 100644
index 000000000..d96d13225
--- /dev/null
+++ b/youtube_dl/extractor/iqiyi.py
@@ -0,0 +1,214 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_chr,
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
+    compat_str,
+)
+
+from ..utils import ExtractorError
+
+import re
+import time
+import json
+import uuid
+import math
+import random
+import zlib
+import hashlib
+
+class IqiyiIE(InfoExtractor):
+    IE_NAME = 'iqiyi'
+
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
+
+    _TEST = {
+            'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
+            'md5': '260f0f59686e65e886995d0ba791ab83',
+            'info_dict': {
+                'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+                'title': '美国德州空中惊现奇异云团 酷似UFO',
+                'ext': 'f4v'
+            }
+    }
+
+    def construct_video_urls(self, data, video_id, _uuid):
+        def do_xor(x, y):
+            a = y % 3
+            if a == 1:
+                return x ^ 121
+            if a == 2:
+                return x ^ 72
+            return x ^ 103
+
+        def get_encode_code(l):
+            a = 0
+            b = l.split('-')
+            c = len(b)
+            s = ''
+            for i in range(c - 1, -1, -1):
+                a = do_xor(int(b[c-i-1], 16), i)
+                s += chr(a)
+            return s[::-1]
+
+        def get_path_key(x):
+            mg = ')(*&^flash@#$%a'
+            tm = self._download_json(
+                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
+            t = str(int(math.floor(int(tm)/(600.0))))
+            return hashlib.md5(
+                 (t+mg+x).encode('utf8')).hexdigest()
+
+        video_urls_dict = {}
+        for i in data['vp']['tkl'][0]['vs']:
+            if 0 < int(i['bid']) <= 10:
+                format_id = self.get_format(i['bid'])
+
+            video_urls_info = i['fs']
+            if not i['fs'][0]['l'].startswith('/'):
+                t = get_encode_code(i['fs'][0]['l'])
+                if t.endswith('mp4'):
+                    video_urls_info = i['flvs']
+
+            video_urls = []
+            for ii in video_urls_info:
+                vl = ii['l']
+                if not vl.startswith('/'):
+                    vl = get_encode_code(vl)
+                key = get_path_key(
+                    vl.split('/')[-1].split('.')[0])
+                filesize = ii['b']
+                base_url = data['vp']['du'].split('/')
+                base_url.insert(-1, key)
+                base_url = '/'.join(base_url)
+                param = {
+                    'su': _uuid,
+                    'qyid': uuid.uuid4().hex,
+                    'client': '',
+                    'z': '',
+                    'bt': '',
+                    'ct': '',
+                    'tn': str(int(time.time()))
+                }
+                api_video_url = base_url + vl + '?' + \
+                    compat_urllib_parse.urlencode(param)
+                js = self._download_json(api_video_url, video_id)
+                video_url = js['l']
+                video_urls.append(
+                    (video_url, filesize))
+
+            video_urls_dict[format_id] = video_urls
+        return video_urls_dict
+
+    def get_format(self, bid):
+        bid_dict = {
+            '1': 'standard',
+            '2': 'high',
+            '3': 'super',
+            '4': 'suprt-high',
+            '5': 'fullhd',
+            '10': '4k'
+        }
+        return bid_dict[str(bid)]
+
+    def get_raw_data(self, tvid, video_id, enc_key, _uuid):
+        tm = str(int(time.time()))
+        param = {
+            'key': 'fvip',
+            'src': hashlib.md5(b'youtube-dl').hexdigest(),
+            'tvId': tvid,
+            'vid': video_id,
+            'vinfo': 1,
+            'tm': tm,
+            'enc': hashlib.md5(
+                (enc_key + tm + tvid).encode('utf8')).hexdigest(),
+            'qyid': _uuid,
+            'tn': random.random(),
+            'um': 0,
+            'authkey': hashlib.md5(
+                (tm + tvid).encode('utf8')).hexdigest()
+        }
+
+        api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
+            compat_urllib_parse.urlencode(param)
+        raw_data = self._download_json(api_url, video_id)
+        return raw_data
+
+    def get_enc_key(self, swf_url, video_id):
+        req = self._request_webpage(
+            swf_url, video_id, note='download swf content')
+        cn = req.read()
+        cn = zlib.decompress(cn[8:])
+        pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
+        enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
+        return enc_key
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(
+            url, 'temp_id', note='download video page')
+        tvid = self._search_regex(
+            r'tvId ?= ?(\'|\")(?P<tvid>\d+)', webpage, 'tvid', flags=re.I, group='tvid')
+        video_id = self._search_regex(
+            r'videoId ?= ?(\'|\")(?P<video_id>[a-z\d]+)',
+            webpage, 'video_id', flags=re.I, group='video_id')
+        swf_url = self._search_regex(
+            r'(?P<swf>http://.+?MainPlayer.+?\.swf)', webpage, 'swf')
+        _uuid = uuid.uuid4().hex
+
+        enc_key = self.get_enc_key(swf_url, video_id)
+
+        raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
+        assert raw_data['code'] == 'A000000'
+        if not raw_data['data']['vp']['tkl']:
+            raise ExtractorError('No support iQiqy VIP video')
+
+        data = raw_data['data']
+
+        title = data['vi']['vn']
+
+        # generate video_urls_dict
+        video_urls_dict = self.construct_video_urls(data, video_id, _uuid)
+
+        # construct info
+        entries = []
+        for format_id in video_urls_dict:
+            video_urls = video_urls_dict[format_id]
+            for i, video_url_info in enumerate(video_urls):
+                if len(entries) < i+1:
+                    entries.append({'formats': []})
+                entries[i]['formats'].append(
+                    {
+                        'url': video_url_info[0],
+                        'filesize': video_url_info[-1],
+                        'format_id': format_id,
+                    }
+                )
+
+        for i in range(len(entries)):
+            entries[i].update(
+                {
+                    'id': '_part%d' % (i+1),
+                    'title': title,
+                }
+            )
+
+        if len(entries) > 1:
+            info = {
+                '_type': 'multi_video',
+                'id': video_id,
+                'title': title,
+                'entries': entries,
+            }
+        else:
+            info = entries[0]
+            info['id'] = video_id
+            info['title'] = title
+
+        return info

From 08f7db20c16743a2bd3040eb7dac11d675011eef Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Sat, 30 May 2015 10:03:32 +0800
Subject: [PATCH 0240/2145] [youku] change format_id

---
 youtube_dl/extractor/youku.py | 37 +++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 063f2e10e..aed6b960a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -132,26 +132,37 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv': '0',
-            'mp4': '1',
-            'hd2': '2',
-            'hd3': '3',
-            '3gp': '0',
-            '3gphd': '1'
+            'flv'   : '0',
+            'mp4'   : '1',
+            'hd2'   : '2',
+            'hd3'   : '3',
+            '3gp'   : '0',
+            '3gphd' : '1'
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
-            'flv': 'flv',
-            'mp4': 'mp4',
-            'hd2': 'flv',
-            'hd3': 'flv',
-            '3gp': 'flv',
-            '3gphd': 'mp4',
+            'flv'   : 'flv',
+            'mp4'   : 'mp4',
+            'hd2'   : 'flv',
+            'hd3'   : 'flv',
+            '3gp'   : 'flv',
+            '3gphd' : 'mp4'
         }
         return ext_dict[fm]
 
+    def get_format_name(self, fm):
+        _dict = {
+            '3gp'   : 'h6',
+            '3gphd' : 'h5',
+            'flv'   : 'h4',
+            'mp4'   : 'h3',
+            'hd2'   : 'h2',
+            'hd3'   : 'h1'
+        }
+        return _dict[fm]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -191,7 +202,7 @@ class YoukuIE(InfoExtractor):
                 entries[i]['formats'].append(
                     {
                         'url': video_urls[i],
-                        'format_id': fm,
+                        'format_id': self.get_format_name(fm),
                         'ext': self.parse_ext_l(fm),
                         'filesize': int(data1['segs'][fm][i]['size'])
                     }

From 670861bd206ab4063baeb6b80d06a054ce4e1d62 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Sat, 30 May 2015 10:37:54 +0800
Subject: [PATCH 0241/2145] [iqiyi] Do not request for unneeded formats

---
 youtube_dl/extractor/iqiyi.py | 72 ++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d96d13225..747f3f902 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -4,20 +4,12 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_chr,
-    compat_parse_qs,
-    compat_urllib_parse,
-    compat_urllib_request,
-    compat_urlparse,
-    compat_str,
-)
+from ..compat import compat_urllib_parse
 
 from ..utils import ExtractorError
 
 import re
 import time
-import json
 import uuid
 import math
 import random
@@ -31,15 +23,15 @@ class IqiyiIE(InfoExtractor):
 
     _TEST = {
             'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-            'md5': '260f0f59686e65e886995d0ba791ab83',
+            'md5': '2cb594dc2781e6c941a110d8f358118b',
             'info_dict': {
                 'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
                 'title': '美国德州空中惊现奇异云团 酷似UFO',
-                'ext': 'f4v'
+                'ext': 'f4v',
             }
     }
 
-    def construct_video_urls(self, data, video_id, _uuid):
+    def construct_video_urls(self, data, video_id, _uuid, bid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -66,10 +58,21 @@ class IqiyiIE(InfoExtractor):
             return hashlib.md5(
                  (t+mg+x).encode('utf8')).hexdigest()
 
+        # get accept format
+        # getting all format will spend minutes for a big video.
+        if bid == 'best':
+            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs'] \
+                   if 0 < int(i['bid']) <= 10]
+            bid = str(max(bids))
+
         video_urls_dict = {}
         for i in data['vp']['tkl'][0]['vs']:
             if 0 < int(i['bid']) <= 10:
                 format_id = self.get_format(i['bid'])
+            else:
+                continue
+
+            video_urls = []
 
             video_urls_info = i['fs']
             if not i['fs'][0]['l'].startswith('/'):
@@ -77,7 +80,12 @@ class IqiyiIE(InfoExtractor):
                 if t.endswith('mp4'):
                     video_urls_info = i['flvs']
 
-            video_urls = []
+            if int(i['bid']) != int(bid):  # ignore missing match format
+                video_urls.extend(
+                    [('http://example.com/v.flv', ii['b']) for ii in video_urls_info])
+                video_urls_dict[format_id] = video_urls
+                continue
+
             for ii in video_urls_info:
                 vl = ii['l']
                 if not vl.startswith('/'):
@@ -108,15 +116,27 @@ class IqiyiIE(InfoExtractor):
         return video_urls_dict
 
     def get_format(self, bid):
-        bid_dict = {
-            '1': 'standard',
-            '2': 'high',
-            '3': 'super',
-            '4': 'suprt-high',
-            '5': 'fullhd',
-            '10': '4k'
+        _dict = {
+            '1'  : 'h6',
+            '2'  : 'h5',
+            '3'  : 'h4',
+            '4'  : 'h3',
+            '5'  : 'h2',
+            '10' : 'h1'
         }
-        return bid_dict[str(bid)]
+        return _dict.get(str(bid), None)
+
+    def get_bid(self, format_id):
+        _dict = {
+            'h6'   : '1',
+            'h5'   : '2',
+            'h4'   : '3',
+            'h3'   : '4',
+            'h2'   : '5',
+            'h1'   : '10',
+            'best' : 'best'
+        }
+        return _dict.get(format_id, None)
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))
@@ -173,8 +193,14 @@ class IqiyiIE(InfoExtractor):
 
         title = data['vi']['vn']
 
+        format = self._downloader.params.get('format', None)
+        bid = self.get_bid(format) if format else 'best'
+        if not bid:
+            raise ExtractorError('Can\'t get format.')
+
         # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data, video_id, _uuid)
+        video_urls_dict = self.construct_video_urls(
+            data, video_id, _uuid, bid)
 
         # construct info
         entries = []
@@ -188,10 +214,12 @@ class IqiyiIE(InfoExtractor):
                         'url': video_url_info[0],
                         'filesize': video_url_info[-1],
                         'format_id': format_id,
+                        'preference': int(self.get_bid(format_id))
                     }
                 )
 
         for i in range(len(entries)):
+            self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
                     'id': '_part%d' % (i+1),

From fafec39d4177f4873bb2393749a46873c4ffda4a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 May 2015 13:23:09 +0800
Subject: [PATCH 0242/2145] [spiegeltv] Changed RTMP server (fixes #5788 and
 fixes #5843)

Thanks to @brickleroux for finding out the problem
---
 youtube_dl/extractor/spiegeltv.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 98cf92d89..359722ad6 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -51,9 +51,9 @@ class SpiegeltvIE(InfoExtractor):
         is_wide = media_json['is_wide']
 
         server_json = self._download_json(
-            'http://www.spiegel.tv/streaming_servers/', video_id,
-            note='Downloading server information')
-        server = server_json[0]['endpoint']
+            'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
+            video_id, note='Downloading server information')
+        server = server_json['streamingserver'][0]['endpoint']
 
         thumbnails = []
         for image in media_json['images']:
@@ -76,5 +76,6 @@ class SpiegeltvIE(InfoExtractor):
             'ext': 'm4v',
             'description': description,
             'duration': duration,
-            'thumbnails': thumbnails
+            'thumbnails': thumbnails,
+            'rtmp_live': True,
         }

From 6ebdfe43e439239df051f6071a23c51705c150cf Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 30 May 2015 09:30:14 +0200
Subject: [PATCH 0243/2145] [tube8] fix extractor (fixes #5846)

---
 youtube_dl/extractor/tube8.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index d73ad3762..6ca8840b0 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -47,7 +47,7 @@ class Tube8IE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
-            r'var flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
+            r'flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
 
         video_url = flashvars['video_url']
         if flashvars.get('encrypted') is True:
@@ -58,19 +58,19 @@ class Tube8IE(InfoExtractor):
         thumbnail = flashvars.get('image_url')
 
         title = self._html_search_regex(
-            r'videotitle\s*=\s*"([^"]+)', webpage, 'title')
+            r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'>Description:</strong>(.+?)<', webpage, 'description', fatal=False)
+            r'>Description:</strong>\s*(.+?)\s*<', webpage, 'description', fatal=False)
         uploader = self._html_search_regex(
-            r'<strong class="video-username">(?:<a href="[^"]+">)?([^<]+)(?:</a>)?</strong>',
+            r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
 
         like_count = int_or_none(self._html_search_regex(
-            r"rupVar\s*=\s*'(\d+)'", webpage, 'like count', fatal=False))
+            r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
         dislike_count = int_or_none(self._html_search_regex(
-            r"rdownVar\s*=\s*'(\d+)'", webpage, 'dislike count', fatal=False))
+            r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
         view_count = self._html_search_regex(
-            r'<strong>Views: </strong>([\d,\.]+)</li>', webpage, 'view count', fatal=False)
+            r'<strong>Views: </strong>([\d,\.]+)\s*</li>', webpage, 'view count', fatal=False)
         if view_count:
             view_count = str_to_int(view_count)
         comment_count = self._html_search_regex(

From 0385d642232ba4e8b455d0c4eb95c7985f22f276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:12:58 +0600
Subject: [PATCH 0244/2145] [crunchyroll] Extract subtitles extraction routine

---
 youtube_dl/extractor/crunchyroll.py | 30 +++++++++++++----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 1c77df47e..4ac537a6d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -76,8 +76,8 @@ class CrunchyrollIE(InfoExtractor):
         self._login()
 
     def _decrypt_subtitles(self, data, iv, id):
-        data = bytes_to_intlist(data)
-        iv = bytes_to_intlist(iv)
+        data = bytes_to_intlist(base64.b64decode(data))
+        iv = bytes_to_intlist(base64.b64decode(iv))
         id = int(id)
 
         def obfuscate_key_aux(count, modulo, start):
@@ -179,6 +179,16 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         return output
 
+    def _extract_subtitles(self, subtitle):
+        sub_root = xml.etree.ElementTree.fromstring(subtitle)
+        return [{
+            'ext': 'srt',
+            'data': self._convert_subtitles_to_srt(sub_root),
+        }, {
+            'ext': 'ass',
+            'data': self._convert_subtitles_to_ass(sub_root),
+        }]
+
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
@@ -190,25 +200,11 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
             if not id or not iv or not data:
                 continue
-            id = int(id)
-            iv = base64.b64decode(iv)
-            data = base64.b64decode(data)
-
             subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            subtitles[lang_code] = [
-                {
-                    'ext': 'srt',
-                    'data': self._convert_subtitles_to_srt(sub_root),
-                },
-                {
-                    'ext': 'ass',
-                    'data': self._convert_subtitles_to_ass(sub_root),
-                },
-            ]
+            subtitles[lang_code] = self._extract_subtitles(subtitle)
         return subtitles
 
     def _real_extract(self, url):

From b2cf6543b21bbe0954c45b35b1402eaca5187c0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:30:04 +0600
Subject: [PATCH 0245/2145] [soompi] Improve and simplify

---
 youtube_dl/extractor/soompi.py | 146 ++++++++++++++++++---------------
 1 file changed, 81 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
index 4726872dc..5da66ca9e 100644
--- a/youtube_dl/extractor/soompi.py
+++ b/youtube_dl/extractor/soompi.py
@@ -2,17 +2,31 @@
 from __future__ import unicode_literals
 
 import re
-import json
-import base64
-import xml.etree.ElementTree
 
-# Soompi uses the same subtitle encryption as crunchyroll
 from .crunchyroll import CrunchyrollIE
 
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    remove_start,
+    xpath_text,
+)
 
-class SoompiIE(CrunchyrollIE):
+
+class SoompiBaseIE(InfoExtractor):
+    def _get_episodes(self, webpage, episode_filter=None):
+        episodes = self._parse_json(
+            self._search_regex(
+                r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'),
+            None)
+        return list(filter(episode_filter, episodes))
+
+
+class SoompiIE(SoompiBaseIE, CrunchyrollIE):
     IE_NAME = 'soompi'
-    _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://tv.soompi.com/en/watch/29235',
         'info_dict': {
@@ -26,84 +40,86 @@ class SoompiIE(CrunchyrollIE):
         },
     }]
 
-    def _get_episodes(self, webpage, episode_filter=None):
-        episodes = json.loads(
-            self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
-        return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
-
-    def _get_subtitles(self, video_id, show_format_xml):
-        subtitles = {}
-        subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
-        subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
+    def _get_episode(self, webpage, video_id):
+        return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0]
 
+    def _get_subtitles(self, config, video_id):
         sub_langs = {}
-        for i in subtitle_info_nodes:
-            sub_langs[i.attrib["id"]] = i.attrib["title"]
+        for subtitle in config.findall('./{default}preload/subtitles/subtitle'):
+            sub_langs[subtitle.attrib['id']] = subtitle.attrib['title']
 
-        for s in subtitle_nodes:
-            lang_code = sub_langs.get(s.attrib["id"], None)
-            if lang_code is None:
+        subtitles = {}
+        for s in config.findall('./{default}preload/subtitle'):
+            lang_code = sub_langs.get(s.attrib['id'])
+            if not lang_code:
+                continue
+            sub_id = s.get('id')
+            data = xpath_text(s, './data', 'data')
+            iv = xpath_text(s, './iv', 'iv')
+            if not id or not iv or not data:
                 continue
-
-            sub_id = int(s.attrib["id"])
-            iv = base64.b64decode(s.find("iv").text)
-            data = base64.b64decode(s.find("data").text)
             subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-
-            subtitles[lang_code] = [{
-                'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
-            }, {
-                'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
-            }]
+            subtitles[lang_code] = self._extract_subtitles(subtitle)
         return subtitles
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            url, video_id, note="Downloading episode page",
-            errnote="Video may not be available for your location")
-        vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage)
-
-        show_meta = json.loads(
-            self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta"))
-        episodes = self._get_episodes(
-            webpage, episode_filter=lambda x: x['id'] == video_id)
-
-        title = episodes[0]["name"]
-        description = episodes[0]["description"]
-        duration = int(episodes[0]["duration"])
-        slug = show_meta["slug"]
+        try:
+            webpage = self._download_webpage(
+                url, video_id, 'Downloading episode page')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                webpage = ee.cause.read()
+                block_message = self._html_search_regex(
+                    r'(?s)<div class="block-message">(.+?)</div>', webpage,
+                    'block message', default=None)
+                if block_message:
+                    raise ExtractorError(block_message, expected=True)
+            raise
 
         formats = []
-        show_format_xml = None
-        for vf in vid_formats:
-            show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \
-                              % (slug, video_id, vf)
-            show_format_xml = self._download_xml(
-                show_format_url, video_id, note="Downloading q%s show xml" % vf)
-            avail_formats = self._extract_m3u8_formats(
-                show_format_xml.find('./{default}preload/stream_info/file').text,
-                video_id, ext="mp4", m3u8_id=vf, preference=int(vf))
-            formats.extend(avail_formats)
+        config = None
+        for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage):
+            config = self._download_xml(
+                'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id),
+                video_id, 'Downloading %s XML' % format_id)
+            m3u8_url = xpath_text(
+                config, './{default}preload/stream_info/file',
+                '%s m3u8 URL' % format_id)
+            if not m3u8_url:
+                continue
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', m3u8_id=format_id))
         self._sort_formats(formats)
 
-        subtitles = self.extract_subtitles(video_id, show_format_xml)
+        episode = self._get_episode(webpage, video_id)
+
+        title = episode['name']
+        description = episode.get('description')
+        duration = int_or_none(episode.get('duration'))
+
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()]
+
+        subtitles = self.extract_subtitles(config, video_id)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnails': thumbnails,
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles
         }
 
 
-class SoompiShowIE(SoompiIE):
+class SoompiShowIE(SoompiBaseIE):
     IE_NAME = 'soompi:show'
-    _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
+    _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
     _TESTS = [{
         'url': 'http://tv.soompi.com/en/shows/liar-game',
         'info_dict': {
@@ -117,14 +133,14 @@ class SoompiShowIE(SoompiIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, show_id, note="Downloading show page")
-        title = self._og_search_title(webpage).replace("SoompiTV | ", "")
+        webpage = self._download_webpage(
+            url, show_id, 'Downloading show page')
+
+        title = remove_start(self._og_search_title(webpage), 'SoompiTV | ')
         description = self._og_search_description(webpage)
 
-        episodes = self._get_episodes(webpage)
-        entries = []
-        for ep in episodes:
-            entries.append(self.url_result(
-                'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
+        entries = [
+            self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi')
+            for episode in self._get_episodes(webpage)]
 
         return self.playlist_result(entries, show_id, title, description)

From 1a5b77dc21384c462e0be86a1638cafd15a6e236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:36:45 +0600
Subject: [PATCH 0246/2145] [crunchyroll] Fix python 3.2

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4ac537a6d..41f0c736d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -76,8 +76,8 @@ class CrunchyrollIE(InfoExtractor):
         self._login()
 
     def _decrypt_subtitles(self, data, iv, id):
-        data = bytes_to_intlist(base64.b64decode(data))
-        iv = bytes_to_intlist(base64.b64decode(iv))
+        data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
+        iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
         id = int(id)
 
         def obfuscate_key_aux(count, modulo, start):

From 5c2191a6053cb5b1210cef68406e8a52e86fd9fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 15:14:10 +0600
Subject: [PATCH 0247/2145] [vgtv] Skip wasLive hds (Closes #5835)

---
 youtube_dl/extractor/vgtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e6ee1e471..654298431 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -107,7 +107,8 @@ class VGTVIE(InfoExtractor):
                 hls_url, video_id, 'mp4', m3u8_id='hls'))
 
         hds_url = streams.get('hds')
-        if hds_url:
+        # wasLive hds are always 404
+        if hds_url and data.get('streamType') != 'wasLive':
             formats.extend(self._extract_f4m_formats(
                 hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 video_id, f4m_id='hds'))

From 4d454c5e4b7ecfae97ff109e05453f43d7cea0a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 15:15:42 +0600
Subject: [PATCH 0248/2145] [vgtv] Check for inactive videos

---
 youtube_dl/extractor/vgtv.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 654298431..472feb7f0 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
 
 
 class VGTVIE(InfoExtractor):
@@ -97,6 +100,10 @@ class VGTVIE(InfoExtractor):
             % (host, video_id, HOST_WEBSITES[host]),
             video_id, 'Downloading media JSON')
 
+        if data.get('status') == 'inactive':
+            raise ExtractorError(
+                'Video %s is no longer available' % video_id, expected=True)
+
         streams = data['streamUrls']
 
         formats = []

From 181c7053e377700c1615bdff2b0fb19235762c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:04:44 +0600
Subject: [PATCH 0249/2145] [YoutubeDL] Make sure all formats have unique
 format_id

---
 youtube_dl/YoutubeDL.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d1953c18f..21d247f23 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1047,6 +1047,8 @@ class YoutubeDL(object):
         if not formats:
             raise ExtractorError('No video formats found!')
 
+        formats_dict = {}
+
         # We check that all the formats have the format and format_id fields
         for i, format in enumerate(formats):
             if 'url' not in format:
@@ -1054,6 +1056,18 @@ class YoutubeDL(object):
 
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
+            format_id = format['format_id']
+            if format_id not in formats_dict:
+                formats_dict[format_id] = []
+            formats_dict[format_id].append(format)
+
+        # Make sure all formats have unique format_id
+        for format_id, ambiguous_formats in formats_dict.items():
+            if len(ambiguous_formats) > 1:
+                for i, format in enumerate(ambiguous_formats):
+                    format['format_id'] = '%s-%d' % (format_id, i)
+
+        for i, format in enumerate(formats):
             if format.get('format') is None:
                 format['format'] = '{id} - {res}{note}'.format(
                     id=format['format_id'],

From b4dd98358f0a68650f6154e8de4e12b8881248aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:12:07 +0600
Subject: [PATCH 0250/2145] [vgtv] Properly handle lives

---
 youtube_dl/extractor/vgtv.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 472feb7f0..f38a72fde 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -62,16 +62,16 @@ class VGTVIE(InfoExtractor):
         },
         {
             # streamType: live
-            'url': 'http://www.vgtv.no/#!/live/100015/direkte-her-kan-du-se-laksen-live-fra-suldalslaagen',
+            'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
             'info_dict': {
-                'id': '100015',
+                'id': '113063',
                 'ext': 'flv',
-                'title': 'DIREKTE: Her kan du se laksen live fra Suldalslågen!',
-                'description': 'md5:9a60cc23fa349f761628924e56eeec2d',
+                'title': 're:^DIREKTE: V75 fra Solvalla [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 0,
-                'timestamp': 1407423348,
-                'upload_date': '20140807',
+                'timestamp': 1432975582,
+                'upload_date': '20150530',
                 'view_count': int,
             },
             'params': {
@@ -105,6 +105,7 @@ class VGTVIE(InfoExtractor):
                 'Video %s is no longer available' % video_id, expected=True)
 
         streams = data['streamUrls']
+        stream_type = data.get('streamType')
 
         formats = []
 
@@ -115,7 +116,7 @@ class VGTVIE(InfoExtractor):
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
-        if hds_url and data.get('streamType') != 'wasLive':
+        if hds_url and stream_type != 'wasLive':
             formats.extend(self._extract_f4m_formats(
                 hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 video_id, f4m_id='hds'))
@@ -143,13 +144,14 @@ class VGTVIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': data['title'],
+            'title': self._live_title(data['title']),
             'description': data['description'],
             'thumbnail': data['images']['main'] + '?t[]=900x506q80',
             'timestamp': data['published'],
             'duration': float_or_none(data['duration'], 1000),
             'view_count': data['displays'],
             'formats': formats,
+            'is_live': True if stream_type == 'live' else False,
         }
 
 

From e6e63e91a70a9c2e4ab92b8afad6fac3b8bede18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:18:11 +0600
Subject: [PATCH 0251/2145] [tf1] Extend _VALID_URL (Closes #5848)

---
 youtube_dl/extractor/tf1.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 656410528..3a68eaa80 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou|www\.tf1)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
@@ -35,6 +35,9 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
         'only_matching': True,
+    }, {
+        'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5196b988971716b9e9c5884d33c757a41aa4548a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 30 May 2015 14:16:18 +0300
Subject: [PATCH 0252/2145] [tubitv] Add new extractor (Closes #5524)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tubitv.py   | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tubitv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f73bf646b..e7e0a55f2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -569,6 +569,7 @@ from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
+from .tubitv import TubiTvIE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
 from .tunein import TuneInIE
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
new file mode 100644
index 000000000..03e971e5e
--- /dev/null
+++ b/youtube_dl/extractor/tubitv.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import codecs
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class TubiTvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video\?id=(?P<id>[0-9]+)'
+    _LOGIN_URL = 'http://tubitv.com/login'
+    _NETRC_MACHINE = 'tubitv'
+    _TEST = {
+        'url': 'http://tubitv.com/video?id=54411&title=The_Kitchen_Musical_-_EP01',
+        'info_dict': {
+            'id': '54411',
+            'ext': 'mp4',
+            'title': 'The Kitchen Musical - EP01',
+            'thumbnail': 're:^https?://.*\.png$',
+            'description': 'md5:37532716166069b353e8866e71fefae7',
+            'duration': 2407,
+        },
+        'params': {
+            'skip_download': 'HLS download',
+        },
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        form_data = {
+            'username': username,
+            'password': password,
+        }
+        payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
+        request = compat_urllib_request.Request(self._LOGIN_URL, payload)
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_page = self._download_webpage(
+            request, None, False, 'Wrong login info')
+        if not re.search(r'id="tubi-logout"', login_page):
+            raise ExtractorError(
+                'Login failed (invalid username/password)', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration'))
+
+        apu = self._search_regex(r"apu='([^']+)'", webpage, 'apu')
+        m3u8_url = codecs.decode(apu, 'rot_13')[::-1]
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+        }

From 1ae7ff771b17d16540aa446aef4f10971465a249 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 30 May 2015 14:33:27 +0300
Subject: [PATCH 0253/2145] [tubitv] Add error message for videos that require
 login (#5524)

---
 youtube_dl/extractor/tubitv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 03e971e5e..2c4b21807 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -59,6 +59,11 @@ class TubiTvIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        if re.search(r"<(?:DIV|div) class='login-required-screen'>", webpage):
+            raise ExtractorError(
+                'This video requires login, use --username and --password '
+                'options to provide account credentials.', expected=True)
+
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)

From 386bdfa698a7f06c43df91913677db3732e29900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 18:29:16 +0600
Subject: [PATCH 0254/2145] [youtube:user] Workaround 35 pages limitation
 (Closes #5778)

---
 youtube_dl/extractor/youtube.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0301682b8..fcdbfe0bc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,6 +1399,26 @@ class YoutubeChannelIE(InfoExtractor):
         channel_id = self._match_id(url)
 
         url = self._TEMPLATE_URL % channel_id
+
+        # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
+        # Workaround by extracting as a playlist if managed to obtain channel playlist URL
+        # otherwise fallback on channel by page extraction
+        channel_page = self._download_webpage(
+            url + '?view=57', channel_id,
+            'Downloading channel page', fatal=False)
+        channel_playlist_id = self._search_regex(
+            [r'<meta itemprop="channelId" content="([^"]+)">',
+             r'data-channel-external-id="([^"]+)"'],
+            channel_page, 'channel id', default=None)
+        if channel_playlist_id and channel_playlist_id.startswith('UC'):
+            playlist_id = 'UU' + channel_playlist_id[2:]
+            channel_playlist = unescapeHTML(self._search_regex(
+                r'href="/?(watch\?v=[0-9A-Za-z_-]{11}&amp;list=%s)"' % playlist_id,
+                channel_page, 'channel playlist URL', default=None))
+            if channel_playlist:
+                return self.url_result(
+                    compat_urlparse.urljoin(url, '/%s' % channel_playlist), 'YoutubePlaylist')
+
         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:

From 9ff811c5cddbf3481fdcd44e97cf3683a925b33f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 May 2015 23:35:55 +0800
Subject: [PATCH 0255/2145] [porn91] PEP8

---
 youtube_dl/extractor/porn91.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index b62eec92d..cdf308f3d 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -5,7 +5,6 @@ import re
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class Porn91IE(InfoExtractor):
@@ -13,13 +12,13 @@ class Porn91IE(InfoExtractor):
     _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
 
     _TEST = {
-            'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-            'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
-            'info_dict': {
-                'id': '7e42283b4f5ab36da134',
-                'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
-                'ext': 'mp4'
-            }
+        'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
+        'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+        'info_dict': {
+            'id': '7e42283b4f5ab36da134',
+            'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+            'ext': 'mp4'
+        }
     }
 
     def _real_extract(self, url):

From 1c2223875664f99325b73fe7765677db9b87e105 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:03:19 +0800
Subject: [PATCH 0256/2145] [porn91] Simplify

---
 youtube_dl/extractor/porn91.py | 38 ++++++++++++++--------------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index cdf308f3d..377ca2c77 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
 
@@ -22,38 +20,34 @@ class Porn91IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
         title = self._search_regex(
-            r'<div id="viewvideo-title">(?P<title>.+?)</div>',
-            webpage, 'title', flags=re.DOTALL)
+            r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')
 
         # get real url
-        n1 = self._search_regex(
-            r'so.addVariable\(\'file\',\'(?P<n1>\d+)\'', webpage, 'n1')
-        n2 = self._search_regex(
-            r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
-        n3 = self._search_regex(
-            r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
+        file_id = self._search_regex(
+            r'so.addVariable\(\'file\',\'(\d+)\'', webpage, 'file id')
+        sec_code = self._search_regex(
+            r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
+        max_vid = self._search_regex(
+            r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
         url_params = compat_urllib_parse.urlencode({
-            'VID': n1,
+            'VID': file_id,
             'mp4': '1',
-            'seccode': n2,
-            'max_vid': n3,
+            'seccode': sec_code,
+            'max_vid': max_vid,
         })
-        t_url = 'http://91porn.com/getfile.php?' + url_params
-        info_cn = self._download_webpage(t_url, video_id, "get real video_url")
-        video_url = self._search_regex(r'file=(?P<url>http.+?)&', info_cn, 'url')
+        info_cn = self._download_webpage(
+            'http://91porn.com/getfile.php?' + url_params, video_id,
+            "get real video url")
+        video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
-        # construct info
-        info = {
+        return {
             'id': video_id,
             'title': title,
             'url': video_url,
         }
-
-        return info

From a80601f8d9789e27c0a916e63d7192c3f398d5d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:20:37 +0800
Subject: [PATCH 0257/2145] [porn91] Extract more info

---
 youtube_dl/extractor/porn91.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 377ca2c77..ea1efc71b 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
 
 
 class Porn91IE(InfoExtractor):
@@ -15,7 +19,8 @@ class Porn91IE(InfoExtractor):
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
-            'ext': 'mp4'
+            'ext': 'mp4',
+            'duration': 431,
         }
     }
 
@@ -46,8 +51,16 @@ class Porn91IE(InfoExtractor):
             "get real video url")
         video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
+        duration = parse_duration(self._search_regex(
+            r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+
+        comment_count = int_or_none(self._search_regex(
+            r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
+
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
+            'duration': duration,
+            'comment_count': comment_count,
         }

From d05a1dbe7013d6314ec477b50d864726e509a872 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:26:12 +0800
Subject: [PATCH 0258/2145] [porn91] Catch daily limit error

---
 youtube_dl/extractor/porn91.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index ea1efc71b..c119c7e94 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -29,6 +30,10 @@ class Porn91IE(InfoExtractor):
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
+
+        if '作为游客，你每天只可观看10个视频' in webpage:
+            raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
+
         title = self._search_regex(
             r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')

From a2d971309b75c79f3f688a0c381707d828cb1026 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:31:18 +0800
Subject: [PATCH 0259/2145] [porn91] Use single quotes

---
 youtube_dl/extractor/porn91.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index c119c7e94..72d1b2718 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -29,7 +29,7 @@ class Porn91IE(InfoExtractor):
         video_id = self._match_id(url)
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
-        webpage = self._download_webpage(url, video_id, "get HTML content")
+        webpage = self._download_webpage(url, video_id, 'get HTML content')
 
         if '作为游客，你每天只可观看10个视频' in webpage:
             raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
@@ -53,7 +53,7 @@ class Porn91IE(InfoExtractor):
         })
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
-            "get real video url")
+            'get real video url')
         video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
         duration = parse_duration(self._search_regex(

From 931bc3c3a719fe33101c05b9fdc4e6ad8eb08bdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 22:52:02 +0600
Subject: [PATCH 0260/2145] [YoutubeDL] Do not loose request method information

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21d247f23..5fc8754c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -49,6 +49,7 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
+    HEADRequest,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
@@ -1720,7 +1721,8 @@ class YoutubeDL(object):
             if req_is_string:
                 req = url_escaped
             else:
-                req = compat_urllib_request.Request(
+                req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+                req = req_type(
                     url_escaped, data=req.data, headers=req.headers,
                     origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
 

From 339516072be6865bf7e9316be81704ae69296c6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 23:16:14 +0600
Subject: [PATCH 0261/2145] [extractor/generic] Unescape video_id and title
 extracted from URL

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9a7b0d25d..c9c92d686 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -894,7 +894,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
+            video_id = os.path.splitext(compat_urllib_parse.unquote(url.rstrip('/').split('/')[-1]))[0]
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -927,7 +927,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(url_basename(url))[0],
+                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
                 'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
@@ -953,7 +953,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(url_basename(url))[0],
+                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,

From 58bde34a236ff98f25fc109a94b3d393f0bbc9ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 00:44:54 +0600
Subject: [PATCH 0262/2145] [extractor/generic] Force Accept-Encoding to any
 for extraction pass

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c9c92d686..ec1d9abbe 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
     compat_xml_parse_error,
+    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
@@ -916,7 +917,9 @@ class GenericIE(InfoExtractor):
 
         full_response = None
         if head_response is False:
-            full_response = self._request_webpage(url, video_id)
+            request = compat_urllib_request.Request(url)
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
             head_response = full_response
 
         # Check for direct link to a video
@@ -941,7 +944,17 @@ class GenericIE(InfoExtractor):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if not full_response:
-            full_response = self._request_webpage(url, video_id)
+            request = compat_urllib_request.Request(url)
+            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
+            # making it impossible to download only chunk of the file (yet we need only 512kB to
+            # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
+            # that will always result in downloading the whole file that is not desirable.
+            # Therefore for extraction pass we have to override Accept-Encoding to any in order
+            # to accept raw bytes and being able to download only a chunk.
+            # It may probably better to solve this by checking Content-Type for application/octet-stream
+            # after HEAD request finishes, but not sure if we can rely on this.
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
 
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!

From 1ddb9456c4a63a207ec40bd74cdf0b36d8c68409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 01:23:58 +0600
Subject: [PATCH 0263/2145] [extractor/generic] Use compat_urllib_parse_unquote
 for unquoting video_id and title from URL

---
 youtube_dl/extractor/generic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec1d9abbe..d9116ce10 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -9,9 +9,10 @@ from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
-    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
@@ -895,7 +896,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = os.path.splitext(compat_urllib_parse.unquote(url.rstrip('/').split('/')[-1]))[0]
+            video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -930,7 +931,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
+                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
@@ -966,7 +967,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
+                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,

From a074e922967fa571d4f1abb1773c711747060f00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:13:24 +0600
Subject: [PATCH 0264/2145] [extractor/generic] Add test for large compressed
 media

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d9116ce10..737141f95 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -138,6 +138,20 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20100513',
             }
         },
+        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        {
+            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
+            'md5': '128c42e68b13950268b648275386fc74',
+            'info_dict': {
+                'id': 'FictionJunction-Parallel_Hearts',
+                'ext': 'flac',
+                'title': 'FictionJunction-Parallel_Hearts',
+                'upload_date': '20140522',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',

From c5fa81fe81ce05cd81c20ff4ea6dac3dccdcbf9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:22:29 +0600
Subject: [PATCH 0265/2145] [extractor/generic] Put all direct link tests near
 to each other for better navigation

---
 youtube_dl/extractor/generic.py | 182 ++++++++++++++++----------------
 1 file changed, 91 insertions(+), 91 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 737141f95..8f2e53063 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,97 @@ class GenericIE(InfoExtractor):
     _VALID_URL = r'.*'
     IE_NAME = 'generic'
     _TESTS = [
+        # Direct link to a video
+        {
+            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
+            'info_dict': {
+                'id': 'trailer',
+                'ext': 'mp4',
+                'title': 'trailer',
+                'upload_date': '20100513',
+            }
+        },
+        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        {
+            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
+            'md5': '128c42e68b13950268b648275386fc74',
+            'info_dict': {
+                'id': 'FictionJunction-Parallel_Hearts',
+                'ext': 'flac',
+                'title': 'FictionJunction-Parallel_Hearts',
+                'upload_date': '20140522',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
+        # Direct download with broken HEAD
+        {
+            'url': 'http://ai-radio.org:8000/radio.opus',
+            'info_dict': {
+                'id': 'radio',
+                'ext': 'opus',
+                'title': 'radio',
+            },
+            'params': {
+                'skip_download': True,  # infinite live stream
+            },
+            'expected_warnings': [
+                r'501.*Not Implemented'
+            ],
+        },
+        # Direct link with incorrect MIME type
+        {
+            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+            'md5': '4ccbebe5f36706d85221f204d7eb5913',
+            'info_dict': {
+                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+                'id': '5_Lennart_Poettering_-_Systemd',
+                'ext': 'webm',
+                'title': '5_Lennart_Poettering_-_Systemd',
+                'upload_date': '20141120',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
+        # RSS feed
+        {
+            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+            'info_dict': {
+                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+                'title': 'Zero Punctuation',
+                'description': 're:.*groundbreaking video review series.*'
+            },
+            'playlist_mincount': 11,
+        },
+        # RSS feed with enclosure
+        {
+            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
+            'info_dict': {
+                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+                'ext': 'm4v',
+                'upload_date': '20150228',
+                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+            }
+        },
+        # google redirect
+        {
+            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
+            'info_dict': {
+                'id': 'cmQHVoWB5FY',
+                'ext': 'mp4',
+                'upload_date': '20130224',
+                'uploader_id': 'TheVerge',
+                'description': 're:^Chris Ziegler takes a look at the\.*',
+                'uploader': 'The Verge',
+                'title': 'First Firefox OS phones side-by-side',
+            },
+            'params': {
+                'skip_download': False,
+            }
+        },
         {
             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
@@ -127,31 +218,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
-        # Direct link to a video
-        {
-            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
-            'info_dict': {
-                'id': 'trailer',
-                'ext': 'mp4',
-                'title': 'trailer',
-                'upload_date': '20100513',
-            }
-        },
-        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
-        {
-            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
-            'md5': '128c42e68b13950268b648275386fc74',
-            'info_dict': {
-                'id': 'FictionJunction-Parallel_Hearts',
-                'ext': 'flac',
-                'title': 'FictionJunction-Parallel_Hearts',
-                'upload_date': '20140522',
-            },
-            'expected_warnings': [
-                'URL could be a direct video link, returning it as such.'
-            ]
-        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
@@ -176,22 +242,6 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
-        # google redirect
-        {
-            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
-            'info_dict': {
-                'id': 'cmQHVoWB5FY',
-                'ext': 'mp4',
-                'upload_date': '20130224',
-                'uploader_id': 'TheVerge',
-                'description': 're:^Chris Ziegler takes a look at the\.*',
-                'uploader': 'The Verge',
-                'title': 'First Firefox OS phones side-by-side',
-            },
-            'params': {
-                'skip_download': False,
-            }
-        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
@@ -423,16 +473,6 @@ class GenericIE(InfoExtractor):
                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
             }
         },
-        # RSS feed
-        {
-            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
-            'info_dict': {
-                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
-                'title': 'Zero Punctuation',
-                'description': 're:.*groundbreaking video review series.*'
-            },
-            'playlist_mincount': 11,
-        },
         # Multiple brightcove videos
         # https://github.com/rg3/youtube-dl/issues/2283
         {
@@ -486,21 +526,6 @@ class GenericIE(InfoExtractor):
                 'uploader': 'thoughtworks.wistia.com',
             },
         },
-        # Direct download with broken HEAD
-        {
-            'url': 'http://ai-radio.org:8000/radio.opus',
-            'info_dict': {
-                'id': 'radio',
-                'ext': 'opus',
-                'title': 'radio',
-            },
-            'params': {
-                'skip_download': True,  # infinite live stream
-            },
-            'expected_warnings': [
-                r'501.*Not Implemented'
-            ],
-        },
         # Soundcloud embed
         {
             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
@@ -532,21 +557,6 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 2,
         },
-        # Direct link with incorrect MIME type
-        {
-            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
-            'md5': '4ccbebe5f36706d85221f204d7eb5913',
-            'info_dict': {
-                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
-                'id': '5_Lennart_Poettering_-_Systemd',
-                'ext': 'webm',
-                'title': '5_Lennart_Poettering_-_Systemd',
-                'upload_date': '20141120',
-            },
-            'expected_warnings': [
-                'URL could be a direct video link, returning it as such.'
-            ]
-        },
         # Cinchcast embed
         {
             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
@@ -705,16 +715,6 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
-        # RSS feed with enclosure
-        {
-            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
-            'info_dict': {
-                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
-                'ext': 'm4v',
-                'upload_date': '20150228',
-                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
-            }
-        },
         # Crooks and Liars embed
         {
             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',

From c5138a7ce49db19b64adc11d81384595b966a7a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:36:20 +0600
Subject: [PATCH 0266/2145] [extractor/generic] Clarify test comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f2e53063..96ca398de 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,7 +59,7 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20100513',
             }
         },
-        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        # Direct link to media delivered compressed (until Accept-Encoding is *)
         {
             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
             'md5': '128c42e68b13950268b648275386fc74',

From d2a9de78dfc629aaaaf8a2a30432d5f02c949e9a Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Sat, 30 May 2015 20:50:22 +0000
Subject: [PATCH 0267/2145] [youtube] Construct a playlist URL in case the page
 is missing one

This fixes jumping from user/channel -> playlist for some users like
https://www.youtube.com/user/BitcoinFoundation

This also removes the superfluous log message
"add --no-playlist to just download video VIDEOID"
when downloading a user/channel.
---
 youtube_dl/extractor/youtube.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fcdbfe0bc..aacb999ce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1412,12 +1412,8 @@ class YoutubeChannelIE(InfoExtractor):
             channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
-            channel_playlist = unescapeHTML(self._search_regex(
-                r'href="/?(watch\?v=[0-9A-Za-z_-]{11}&amp;list=%s)"' % playlist_id,
-                channel_page, 'channel playlist URL', default=None))
-            if channel_playlist:
-                return self.url_result(
-                    compat_urlparse.urljoin(url, '/%s' % channel_playlist), 'YoutubePlaylist')
+            return self.url_result(
+                compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
 
         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)

From eb47569f8a6017190d73429b3ef54c1ffaf201dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 03:00:13 +0600
Subject: [PATCH 0268/2145] [tvigle] Add support for m3u8

---
 youtube_dl/extractor/tvigle.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 102362b29..4e95bd30f 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
+    int_or_none,
     parse_age_limit,
 )
 
@@ -24,17 +25,17 @@ class TvigleIE(InfoExtractor):
                 'display_id': 'sokrat',
                 'ext': 'flv',
                 'title': 'Сократ',
-                'description': 'md5:a05bd01be310074d5833efc6743be95e',
+                'description': 'md5:d6b92ffb7217b4b8ebad2e7665253c17',
                 'duration': 6586,
-                'age_limit': 0,
+                'age_limit': 12,
             },
         },
         {
             'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
-            'md5': 'd9012d7c7c598fe7a11d7fb46dc1f574',
+            'md5': 'e7efe5350dd5011d0de6550b53c3ba7b',
             'info_dict': {
                 'id': '5142516',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Ведущий телепрограммы «60 минут» (США) о Владимире Высоцком',
                 'description': 'md5:027f7dc872948f14c96d19b4178428a4',
                 'duration': 186.080,
@@ -54,7 +55,7 @@ class TvigleIE(InfoExtractor):
         if not video_id:
             webpage = self._download_webpage(url, display_id)
             video_id = self._html_search_regex(
-                r'<li class="video-preview current_playing" id="(\d+)">',
+                r'class="video-preview current_playing" id="(\d+)">',
                 webpage, 'video id')
 
         video_data = self._download_json(
@@ -70,13 +71,19 @@ class TvigleIE(InfoExtractor):
 
         formats = []
         for vcodec, fmts in item['videos'].items():
-            for quality, video_url in fmts.items():
+            for format_id, video_url in fmts.items():
+                if format_id == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=vcodec))
+                    continue
+                height = self._search_regex(
+                    r'^(\d+)[pP]$', format_id, 'height', default=None)
                 formats.append({
                     'url': video_url,
-                    'format_id': '%s-%s' % (vcodec, quality),
+                    'format_id': '%s-%s' % (vcodec, format_id),
                     'vcodec': vcodec,
-                    'height': int(quality[:-1]),
-                    'filesize': item['video_files_size'][vcodec][quality],
+                    'height': int_or_none(height),
+                    'filesize': item['video_files_size'][vcodec][format_id],
                 })
         self._sort_formats(formats)
 

From 7584e38ce4e98e0e9abca146a513d215701308e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 03:01:41 +0600
Subject: [PATCH 0269/2145] [tvigle] Modernize

---
 youtube_dl/extractor/tvigle.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 4e95bd30f..a85693888 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -64,8 +64,8 @@ class TvigleIE(InfoExtractor):
         item = video_data['playlist']['items'][0]
 
         title = item['title']
-        description = item['description']
-        thumbnail = item['thumbnail']
+        description = item.get('description')
+        thumbnail = item.get('thumbnail')
         duration = float_or_none(item.get('durationMilliseconds'), 1000)
         age_limit = parse_age_limit(item.get('ageRestrictions'))
 
@@ -83,7 +83,7 @@ class TvigleIE(InfoExtractor):
                     'format_id': '%s-%s' % (vcodec, format_id),
                     'vcodec': vcodec,
                     'height': int_or_none(height),
-                    'filesize': item['video_files_size'][vcodec][format_id],
+                    'filesize': int_or_none(item.get('video_files_size', {}).get(vcodec, {}).get(format_id)),
                 })
         self._sort_formats(formats)
 

From df15ef8dab6df79eb076b3d06b3948917763ac3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 04:05:09 +0600
Subject: [PATCH 0270/2145] [YoutubeDL] Tweak select_format for video only
 media

---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5fc8754c6..aa6ec9d9a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -924,8 +924,9 @@ class YoutubeDL(object):
                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
             if audiovideo_formats:
                 return audiovideo_formats[format_idx]
-            # for audio only urls, select the best/worst audio format
-            elif all(f.get('acodec') != 'none' for f in available_formats):
+            # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
+            elif (all(f.get('acodec') != 'none' for f in available_formats) or
+                  all(f.get('vcodec') != 'none' for f in available_formats)):
                 return available_formats[format_idx]
         elif format_spec == 'bestaudio':
             audio_formats = [

From 96b9690985e9b9f4e50fde10bbc92e1a72df64e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 04:05:26 +0600
Subject: [PATCH 0271/2145] [imgur] Improve extraction

---
 youtube_dl/extractor/imgur.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index fe5d95e2c..d692ea79a 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
@@ -12,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -34,7 +35,8 @@ class ImgurIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            compat_urlparse.urljoin(url, video_id), video_id)
 
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',

From 47fd8c2f761c2073744cb041f9eccb7ed10f2470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jun 2015 00:04:36 +0600
Subject: [PATCH 0272/2145] [patreon] Fix embeds extraction (Closes #5862)

---
 youtube_dl/extractor/patreon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index f179ea200..6cdc2638b 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -87,7 +87,7 @@ class PatreonIE(InfoExtractor):
             r'<div class="attach"><a target="_blank" href="([^"]+)">',
             webpage, 'attachment URL', default=None)
         embed = self._html_search_regex(
-            r'<div id="watchCreation">\s*<iframe class="embedly-embed" src="([^"]+)"',
+            r'<div[^>]+id="watchCreation"[^>]*>\s*<iframe[^>]+src="([^"]+)"',
             webpage, 'embedded URL', default=None)
 
         if attach_fn is not None:

From 4053ee9104cd7669f749a267dc2c2a1725ca188b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Jun 2015 14:43:20 +0800
Subject: [PATCH 0273/2145] Credit @PeterDing for 91porn extractor (#5830)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3410e1fb9..bf2a25cb8 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -126,3 +126,4 @@ Matthias Küch
 Julian Richen
 Ping O.
 Mister Hat
+Peter Ding

From 866b296d0f156831cceccc967c34382a90b77422 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 1 Jun 2015 16:11:19 +0300
Subject: [PATCH 0274/2145] [aftonbladet] Fix extraction and update _VALID_URL
 (Fixes #5863)

---
 youtube_dl/extractor/aftonbladet.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index a117502bc..e0518cf26 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -6,11 +6,11 @@ from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.aftonbladet\.se/webbtv.+?(?P<id>article[0-9]+)\.ab(?:$|[?#])'
+    _VALID_URL = r'http://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://tv.aftonbladet.se/webbtv/nyheter/vetenskap/rymden/article36015.ab',
+        'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
         'info_dict': {
-            'id': 'article36015',
+            'id': '36015',
             'ext': 'mp4',
             'title': 'Vulkanutbrott i rymden - nu släpper NASA bilderna',
             'description': 'Jupiters måne mest aktiv av alla himlakroppar',
@@ -25,8 +25,9 @@ class AftonbladetIE(InfoExtractor):
 
         # find internal video meta data
         meta_url = 'http://aftonbladet-play.drlib.aptoma.no/video/%s.json'
-        internal_meta_id = self._html_search_regex(
-            r'data-aptomaId="([\w\d]+)"', webpage, 'internal_meta_id')
+        player_config = self._parse_json(self._html_search_regex(
+            r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
+        internal_meta_id = player_config['videoId']
         internal_meta_url = meta_url % internal_meta_id
         internal_meta_json = self._download_json(
             internal_meta_url, video_id, 'Downloading video meta data')

From 923e79e2e4d9cc0c24496614aab520737cdc89ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 00:53:04 +0600
Subject: [PATCH 0275/2145] [nova] Add extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/nova.py     | 135 +++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 youtube_dl/extractor/nova.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4dc07efe0..67eb96057 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -352,6 +352,7 @@ from .ninegag import NineGagIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
+from .nova import NovaIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
 from .nowtv import NowTVIE
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
new file mode 100644
index 000000000..e93a7ffa8
--- /dev/null
+++ b/youtube_dl/extractor/nova.py
@@ -0,0 +1,135 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class NovaIE(InfoExtractor):
+    IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+)(?:\.html|/?)'
+    _TESTS = [{
+        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
+        'info_dict': {
+            'id': '1608920',
+            'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
+            'ext': 'flv',
+            'title': 'Duel: Michal Hrdlička a Petr Suchoň',
+            'description': 'md5:d0cc509858eee1b1374111c588c6f5d5',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html',
+        'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
+        'info_dict': {
+            'id': '1757139',
+            'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci',
+            'ext': 'mp4',
+            'title': 'Podzemní nemocnice v pražské Krči',
+            'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        }
+    }, {
+        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/',
+        'info_dict': {
+            'id': '1756825',
+            'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
+            'ext': 'flv',
+            'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
+            'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://doma.nova.cz/clanek/zdravi/prijdte-se-zapsat-do-registru-kostni-drene-jiz-ve-stredu-3-cervna.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        site = mobj.group('site')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            [r"(?:media|video_id)\s*:\s*'(\d+)'",
+             r'media=(\d+)',
+             r'id="article_video_(\d+)"',
+             r'id="player_(\d+)"'],
+            webpage, 'video id')
+
+        config_url = self._search_regex(
+            r'src="(http://tn\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
+            webpage, 'config url', default=None)
+
+        if not config_url:
+            DEFAULT_SITE_ID = '23000'
+            SITES = {
+                'tvnoviny': DEFAULT_SITE_ID,
+                'novaplus': DEFAULT_SITE_ID,
+                'vymena': DEFAULT_SITE_ID,
+                'krasna': DEFAULT_SITE_ID,
+                'fanda': '30',
+                'tn': '30',
+                'doma': '30',
+            }
+
+            site_id = self._search_regex(
+                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(site, DEFAULT_SITE_ID)
+
+            config_url = ('http://tn.nova.cz/bin/player/videojs/config.php?site=%s&media=%s&jsVar=vjsconfig'
+                          % (site_id, video_id))
+
+        config = self._download_json(
+            config_url, display_id,
+            'Downloading config JSON',
+            transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s))
+
+        mediafile = config['mediafile']
+        video_url = mediafile['src']
+
+        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
+        if m:
+            formats = [{
+                'url': m.group('url'),
+                'app': m.group('app'),
+                'play_path': m.group('playpath'),
+                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
+                'ext': 'flv',
+            }]
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+        self._sort_formats(formats)
+
+        title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = config.get('poster')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 60158217ef8da5f44ef316e50c8a5e2ac2e202c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 00:57:08 +0600
Subject: [PATCH 0276/2145] [nova] Add tv test

---
 youtube_dl/extractor/nova.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index e93a7ffa8..4e999b237 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -60,6 +60,9 @@ class NovaIE(InfoExtractor):
     }, {
         'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
         'only_matching': True,
+    }, {
+        'url': 'http://tv.nova.cz/clanek/novinky/zivot-je-zivot-bondovsky-trailer.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9f4b9118ccaef5cd7c414a78c5622968e8c3343f Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:47:52 +0300
Subject: [PATCH 0277/2145] [nova] Fix display_id extraction bug

Make id group non-greedy so that .html is not included in it.
---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 4e999b237..1dd18511e 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+)(?:\.html|/?)'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/?)$'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
         'info_dict': {

From 9464a194dbf48989c486fa2de9e1aebc59e28ed4 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:52:39 +0300
Subject: [PATCH 0278/2145] [nova] Fix extension extraction bug

Replace the hardcoded flv with determine_ext. Let rtmpdump parse the url.
---
 youtube_dl/extractor/nova.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 1dd18511e..fd5f9cb0e 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class NovaIE(InfoExtractor):
@@ -39,7 +40,7 @@ class NovaIE(InfoExtractor):
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
             'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
@@ -108,21 +109,8 @@ class NovaIE(InfoExtractor):
 
         mediafile = config['mediafile']
         video_url = mediafile['src']
-
-        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
-        if m:
-            formats = [{
-                'url': m.group('url'),
-                'app': m.group('app'),
-                'play_path': m.group('playpath'),
-                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
-                'ext': 'flv',
-            }]
-        else:
-            formats = [{
-                'url': video_url,
-            }]
-        self._sort_formats(formats)
+        ext = determine_ext(video_url)
+        video_url = video_url.replace('&{}:'.format(ext), '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = self._og_search_description(webpage)
@@ -134,5 +122,6 @@ class NovaIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'formats': formats,
+            'url': video_url,
+            'ext': ext,
         }

From fcb04bcaca1b83cd3f13f494d7d775e35e0b6182 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:55:41 +0300
Subject: [PATCH 0279/2145] [nova] Extract upload_date in some cases

---
 youtube_dl/extractor/nova.py | 39 ++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index fd5f9cb0e..30c64aaf8 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -49,6 +49,33 @@ class NovaIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
+        'info_dict': {
+            'id': '1756858',
+            'ext': 'mp4',
+            'title': 'Televizní noviny - 30. 5. 2015',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+            'upload_date': '20150530',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
+        'info_dict': {
+            'id': '1753621',
+            'ext': 'mp4',
+            'title': 'Zaklínač 3: Divoký hon',
+            'description': 're:.*Pokud se stejně jako my nemůžete.*',
+            'thumbnail': 're:https?://.*\.jpg(\?.*)?',
+            'upload_date': '20150521',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
     }, {
         'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
         'only_matching': True,
@@ -116,11 +143,23 @@ class NovaIE(InfoExtractor):
         description = self._og_search_description(webpage)
         thumbnail = config.get('poster')
 
+        mobj = None
+        if site == 'novaplus':
+            mobj = re.search(r'(?P<day>\d{1,2})-(?P<month>\d{1,2})-(?P<year>\d{4})$', display_id)
+        if site == 'fanda':
+            mobj = re.search(
+                r'<span class="date_time">(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4})\b', webpage)
+        if mobj:
+            upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day')))
+        else:
+            upload_date = None
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
+            'upload_date': upload_date,
             'thumbnail': thumbnail,
             'url': video_url,
             'ext': ext,

From 34c0f95db273ac5e7a7f8a6d23a3f90ceadf4695 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:56:36 +0300
Subject: [PATCH 0280/2145] [nova] Remove html tags from description

---
 youtube_dl/extractor/nova.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 30c64aaf8..140312f9c 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import clean_html, determine_ext
 
 
 class NovaIE(InfoExtractor):
@@ -42,7 +42,7 @@ class NovaIE(InfoExtractor):
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
             'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
+            'description': 'md5:dc24e50be5908df83348e50d1431295e',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         },
         'params': {
@@ -140,7 +140,7 @@ class NovaIE(InfoExtractor):
         video_url = video_url.replace('&{}:'.format(ext), '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
+        description = clean_html(self._og_search_description(webpage, default=None))
         thumbnail = config.get('poster')
 
         mobj = None

From a00234f1c517d077a237da576be638fef980d79e Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:57:03 +0300
Subject: [PATCH 0281/2145] [nova] Minor style improvement

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 140312f9c..10957e5fa 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -132,7 +132,7 @@ class NovaIE(InfoExtractor):
         config = self._download_json(
             config_url, display_id,
             'Downloading config JSON',
-            transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s))
+            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
 
         mediafile = config['mediafile']
         video_url = mediafile['src']

From bc03e58565d99677a643e0a058d25c7ee9b265d6 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 13:16:58 +0300
Subject: [PATCH 0282/2145] [iprima] Update

---
 youtube_dl/extractor/iprima.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 8529bedfc..f3536893a 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/[^?#]+/(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/([^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/particka/particka-92',
@@ -23,7 +23,7 @@ class IPrimaIE(InfoExtractor):
             'id': '39152',
             'ext': 'flv',
             'title': 'Partička (92)',
-            'description': 'md5:3740fda51464da35a2d4d0670b8e4fd6',
+            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
         },
         'params': {
@@ -35,13 +35,16 @@ class IPrimaIE(InfoExtractor):
             'id': '9718337',
             'ext': 'flv',
             'title': 'Tchibo Partička - Jarní móda',
-            'description': 'md5:589f8f59f414220621ff8882eb3ce7be',
+            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,  # requires rtmpdump
         },
         'skip': 'Do not have permission to access this page',
+    }, {
+        'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -102,7 +105,7 @@ class IPrimaIE(InfoExtractor):
 
         return {
             'id': real_id,
-            'title': self._og_search_title(webpage),
+            'title': self._og_search_title(webpage).replace(' | Prima PLAY', ''),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'description': self._og_search_description(webpage),

From b5597738d4de35fd6f2be7bf1cb6a32c754d873f Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 17:28:14 +0300
Subject: [PATCH 0283/2145] [iprima] Comply with review

---
 youtube_dl/extractor/iprima.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index f3536893a..502507551 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -11,11 +11,12 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    remove_end,
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/([^/]+/)*(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/particka/particka-92',
@@ -105,7 +106,7 @@ class IPrimaIE(InfoExtractor):
 
         return {
             'id': real_id,
-            'title': self._og_search_title(webpage).replace(' | Prima PLAY', ''),
+            'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'description': self._og_search_description(webpage),

From d23da75b32e02963f988bad962b3f5259e4a6d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 21:10:18 +0600
Subject: [PATCH 0284/2145] [iprima] Fix description extraction

`og:description` does not contain actual description anymore.
---
 youtube_dl/extractor/iprima.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 502507551..821c8ec10 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -24,7 +24,7 @@ class IPrimaIE(InfoExtractor):
             'id': '39152',
             'ext': 'flv',
             'title': 'Partička (92)',
-            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
+            'description': 'md5:74e9617e51bca67c3ecfb2c6f9766f45',
             'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
         },
         'params': {
@@ -36,13 +36,11 @@ class IPrimaIE(InfoExtractor):
             'id': '9718337',
             'ext': 'flv',
             'title': 'Tchibo Partička - Jarní móda',
-            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,  # requires rtmpdump
         },
-        'skip': 'Do not have permission to access this page',
     }, {
         'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
         'only_matching': True,
@@ -109,5 +107,7 @@ class IPrimaIE(InfoExtractor):
             'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
-            'description': self._og_search_description(webpage),
+            'description': self._search_regex(
+                r'<p[^>]+itemprop="description"[^>]*>([^<]+)',
+                webpage, 'description', default=None),
         }

From 4b5fe1349f5568f3b9b939520db0a1ddc598b4b3 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 18:15:05 +0300
Subject: [PATCH 0285/2145] [nova] Comply with review

---
 youtube_dl/extractor/nova.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 10957e5fa..4a2d76506 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import clean_html, determine_ext
+from ..utils import (
+    clean_html,
+    determine_ext,
+    unified_strdate,
+)
 
 
 class NovaIE(InfoExtractor):
@@ -143,14 +147,12 @@ class NovaIE(InfoExtractor):
         description = clean_html(self._og_search_description(webpage, default=None))
         thumbnail = config.get('poster')
 
-        mobj = None
         if site == 'novaplus':
-            mobj = re.search(r'(?P<day>\d{1,2})-(?P<month>\d{1,2})-(?P<year>\d{4})$', display_id)
-        if site == 'fanda':
-            mobj = re.search(
-                r'<span class="date_time">(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4})\b', webpage)
-        if mobj:
-            upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day')))
+            upload_date = unified_strdate(self._search_regex(
+                r'(\d{1,2}-\d{1,2}-\d{4})$', display_id, 'upload date', default=None))
+        elif site == 'fanda':
+            upload_date = unified_strdate(self._search_regex(
+                r'<span class="date_time">(\d{1,2}\.\d{1,2}\.\d{4})', webpage, 'upload date', default=None))
         else:
             upload_date = None
 

From 08b7968e2873b45dafe465ec04541db8fcd4967d Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 17:49:15 +0300
Subject: [PATCH 0286/2145] [nova] Fix display_id extraction bug

---
 youtube_dl/extractor/nova.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 4a2d76506..8360a65d9 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -13,9 +13,9 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/?)$'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
-        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
+        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
             'id': '1608920',
             'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
@@ -29,7 +29,7 @@ class NovaIE(InfoExtractor):
             'skip_download': True,
         }
     }, {
-        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html',
+        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
         'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
         'info_dict': {
             'id': '1757139',
@@ -40,7 +40,7 @@ class NovaIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         }
     }, {
-        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/',
+        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove',
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',

From b0cda32f726443d464a68a34b22a2e02ef8b29b0 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 18:17:33 +0300
Subject: [PATCH 0287/2145] [nova] Fix Python 2.6 compatability issue

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 8360a65d9..7e3498eea 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -141,7 +141,7 @@ class NovaIE(InfoExtractor):
         mediafile = config['mediafile']
         video_url = mediafile['src']
         ext = determine_ext(video_url)
-        video_url = video_url.replace('&{}:'.format(ext), '')
+        video_url = video_url.replace('&%s:' % ext, '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = clean_html(self._og_search_description(webpage, default=None))

From fa971259e69a8031c384754b6238cfff71bea773 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 19:09:47 +0300
Subject: [PATCH 0288/2145] [nova] Add a comment about html in description

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 7e3498eea..85253b6ed 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -46,7 +46,7 @@ class NovaIE(InfoExtractor):
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
             'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:dc24e50be5908df83348e50d1431295e',
+            'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         },
         'params': {

From 23dd1fc74c36329ed40855301ac499a0ad2a0009 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 10:21:03 +0800
Subject: [PATCH 0289/2145] [vidme] Always use the non-embedded page

For example, https://vid.me/Wmur contains more information than
https://vid.me/e/Wmur
---
 youtube_dl/extractor/vidme.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index bd953fb4c..e0b55078b 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vid.me/QNB',
         'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
         'info_dict': {
@@ -23,9 +23,14 @@ class VidmeIE(InfoExtractor):
             'upload_date': '20140725',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-    }
+    }, {
+        # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
+        'url': 'https://vid.me/e/Wmur',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
+        url = url.replace('vid.me/e/', 'vid.me/')
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 

From 8f9478412424b87e4fb77be53d239c13932b078a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 10:26:39 +0800
Subject: [PATCH 0290/2145] [tumblr] Detect vid.me embeds (fixes #5883)

---
 youtube_dl/extractor/tumblr.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 828c808a6..e6218808f 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -28,6 +28,17 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
+        'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
+        'info_dict': {
+            'id': 'Wmur',
+            'ext': 'mp4',
+            'title': 'naked smoking & stretching',
+            'upload_date': '20150506',
+            'timestamp': 1430931613,
+        },
+        'add_ie': ['Vidme'],
     }]
 
     def _real_extract(self, url):
@@ -38,6 +49,12 @@ class TumblrIE(InfoExtractor):
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage = self._download_webpage(url, video_id)
 
+        vid_me_embed_url = self._search_regex(
+            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
+            webpage, 'vid.me embed', default=None)
+        if vid_me_embed_url is not None:
+            return self.url_result(vid_me_embed_url, 'Vidme')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From 687cb3ad35ac49f1053c1ea52e3b6db18b3aa1cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:47:11 +0600
Subject: [PATCH 0291/2145] [24video] Fix uploader extraction

---
 youtube_dl/extractor/twentyfourvideo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 67e8bfea0..c1ee1decc 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -15,7 +15,7 @@ class TwentyFourVideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.24video.net/video/view/1044982',
-            'md5': '48dd7646775690a80447a8dca6a2df76',
+            'md5': 'd041af8b5b4246ea466226a0d6693345',
             'info_dict': {
                 'id': '1044982',
                 'ext': 'mp4',
@@ -54,7 +54,7 @@ class TwentyFourVideoIE(InfoExtractor):
             webpage, 'upload date'))
 
         uploader = self._html_search_regex(
-            r'Загрузил\s*<a href="/jsecUser/movies/[^"]+" class="link">([^<]+)</a>',
+            r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
             webpage, 'uploader', fatal=False)
 
         view_count = int_or_none(self._html_search_regex(

From 15b74b94beb8720a4f6d7ee076c123dd8ae05309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:52:47 +0600
Subject: [PATCH 0292/2145] [tvigle] Capture error message

---
 youtube_dl/extractor/tvigle.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index a85693888..aa07c8251 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     float_or_none,
     int_or_none,
     parse_age_limit,
@@ -63,6 +64,13 @@ class TvigleIE(InfoExtractor):
 
         item = video_data['playlist']['items'][0]
 
+        videos = item.get('videos')
+
+        error_message = item.get('errorMessage')
+        if not videos and error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
         title = item['title']
         description = item.get('description')
         thumbnail = item.get('thumbnail')

From 3153a2c98d7201b1ae8104c346db58e19f322cc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:53:54 +0600
Subject: [PATCH 0293/2145] [tvigle] Skip tests

---
 youtube_dl/extractor/tvigle.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index aa07c8251..dc3a8334a 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -30,6 +30,7 @@ class TvigleIE(InfoExtractor):
                 'duration': 6586,
                 'age_limit': 12,
             },
+            'skip': 'georestricted',
         },
         {
             'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
@@ -42,6 +43,7 @@ class TvigleIE(InfoExtractor):
                 'duration': 186.080,
                 'age_limit': 0,
             },
+            'skip': 'georestricted',
         }, {
             'url': 'https://cloud.tvigle.ru/video/5267604/',
             'only_matching': True,

From 6800d3372f35e08dcc4d34d06601815bf0cb0a3d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 23:10:18 +0800
Subject: [PATCH 0294/2145] [YoutubeDL] Support DASH manifest downloading

---
 youtube_dl/downloader/dash.py   | 50 +++++++++++++++++++++++++++++++++
 youtube_dl/downloader/http.py   |  4 +++
 youtube_dl/extractor/youtube.py |  6 ++++
 3 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/downloader/dash.py

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
new file mode 100644
index 000000000..18eca2c04
--- /dev/null
+++ b/youtube_dl/downloader/dash.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
+import re
+
+
+class DashSegmentsFD(FileDownloader):
+    """
+    Download segments in a DASH manifest
+    """
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+        base_url = info_dict['url']
+        segment_urls = info_dict['segment_urls']
+
+        self.byte_counter = 0
+
+        def append_url_to_file(outf, target_url, target_name):
+            self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
+            req = compat_urllib_request.Request(target_url)
+            data = self.ydl.urlopen(req).read()
+            outf.write(data)
+            self.byte_counter += len(data)
+
+        def combine_url(base_url, target_url):
+            if re.match(r'^https?://', target_url):
+                return target_url
+            return '%s/%s' % (base_url, target_url)
+
+        with open(tmpfilename, 'wb') as outf:
+            append_url_to_file(
+                outf, combine_url(base_url, info_dict['initialization_url']),
+                'initialization segment')
+            for i, segment_url in enumerate(segment_urls):
+                append_url_to_file(
+                    outf, combine_url(base_url, segment_url),
+                    'segment %d / %d' % (i + 1, len(segment_urls)))
+
+        self.try_rename(tmpfilename, filename)
+
+        self._hook_progress({
+            'downloaded_bytes': self.byte_counter,
+            'total_bytes': self.byte_counter,
+            'filename': filename,
+            'status': 'finished',
+        })
+
+        return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b7f144af9..ceacb8522 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -6,6 +6,7 @@ import socket
 import time
 
 from .common import FileDownloader
+from .dash import DashSegmentsFD
 from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
@@ -19,6 +20,9 @@ from ..utils import (
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
+        if info_dict.get('initialization_url') and list(filter(None, info_dict.get('segment_urls', []))):
+            return DashSegmentsFD(self.ydl, self.params).real_download(filename, info_dict)
+
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
         stream = None
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aacb999ce..5d1297e0d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -802,6 +802,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # TODO implement WebVTT downloading
                     pass
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
                     format_id = r.attrib['id']
                     video_url = url_el.text
                     filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
@@ -815,6 +816,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
+                    if segment_list:
+                        f.update({
+                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')]
+                        })
                     try:
                         existing_format = next(
                             fo for fo in formats

From e4ac7bb1e598b0317742737ea06c162fa7f22cd4 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Wed, 3 Jun 2015 19:18:41 +0300
Subject: [PATCH 0295/2145] [nova] Revert "Fix extension extraction bug"

This reverts commit 9464a194dbf48989c486fa2de9e1aebc59e28ed4.
---
 youtube_dl/extractor/nova.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 85253b6ed..3f9c776ef 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
-    determine_ext,
     unified_strdate,
 )
 
@@ -44,7 +43,7 @@ class NovaIE(InfoExtractor):
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
             'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
             'thumbnail': 're:^https?://.*\.(?:jpg)',
@@ -57,7 +56,7 @@ class NovaIE(InfoExtractor):
         'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
         'info_dict': {
             'id': '1756858',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Televizní noviny - 30. 5. 2015',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
             'upload_date': '20150530',
@@ -140,8 +139,21 @@ class NovaIE(InfoExtractor):
 
         mediafile = config['mediafile']
         video_url = mediafile['src']
-        ext = determine_ext(video_url)
-        video_url = video_url.replace('&%s:' % ext, '')
+
+        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
+        if m:
+            formats = [{
+                'url': m.group('url'),
+                'app': m.group('app'),
+                'play_path': m.group('playpath'),
+                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
+                'ext': 'flv',
+            }]
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+        self._sort_formats(formats)
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = clean_html(self._og_search_description(webpage, default=None))
@@ -163,6 +175,5 @@ class NovaIE(InfoExtractor):
             'description': description,
             'upload_date': upload_date,
             'thumbnail': thumbnail,
-            'url': video_url,
-            'ext': ext,
+            'formats': formats,
         }

From 4c8fea92f350d5a3f33d505980ac750b05a9cd34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Jun 2015 23:50:38 +0200
Subject: [PATCH 0296/2145] [test/aes] Fix on python 3.3 and higher

Since 878563c847fa5248eedbd44187536dec04643eaf the aes functions only accepts the base64 data as a unicode string.
---
 test/test_aes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_aes.py b/test/test_aes.py
index 4dc7de7b5..315a3f5ae 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -39,7 +39,7 @@ class TestAES(unittest.TestCase):
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8]) +
             b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
-        )
+        ).decode('utf-8')
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
@@ -47,7 +47,7 @@ class TestAES(unittest.TestCase):
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8]) +
             b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
-        )
+        ).decode('utf-8')
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 

From eedda32e6bc4620d704eabab7a2c8f4b1f1a9169 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 11:27:18 +0800
Subject: [PATCH 0297/2145] [qqmusic] Fix toplist

---
 youtube_dl/extractor/qqmusic.py | 58 +++++++++++++--------------------
 1 file changed, 23 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index b540033e2..48f28ffe9 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -181,60 +181,48 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=toplist&p=global_12',
+        'url': 'http://y.qq.com/#type=toplist&p=global_123',
         'info_dict': {
-            'id': 'global_12',
-            'title': 'itunes榜',
+            'id': 'global_123',
+            'title': '美国iTunes榜',
         },
         'playlist_count': 10,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=top_6',
+        'url': 'http://y.qq.com/#type=toplist&p=top_3',
         'info_dict': {
-            'id': 'top_6',
+            'id': 'top_3',
             'title': 'QQ音乐巅峰榜·欧美',
+            'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
+                           '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
+                           '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'
+                           '登录用户完整播放一首歌曲，记为一次有效播放；同一用户收听同一首歌曲，每天记录为1次有效播放'
         },
         'playlist_count': 100,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=global_5',
+        'url': 'http://y.qq.com/#type=toplist&p=global_106',
         'info_dict': {
-            'id': 'global_5',
-            'title': '韩国mnet排行榜',
+            'id': 'global_106',
+            'title': '韩国Mnet榜',
         },
         'playlist_count': 50,
     }]
 
-    @staticmethod
-    def strip_qq_jsonp(code):
-        return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
         list_type, num_id = list_id.split("_")
 
-        list_page = self._download_webpage(
-            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
+        toplist_json = self._download_json(
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg?type=%s&topid=%s&format=json'
+            % (list_type, num_id),
             list_id, 'Download toplist page')
 
-        entries = []
-        if list_type == 'top':
-            jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
-        else:
-            jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['data']['songmid'], 'QQMusic', song['data']['songmid']
+            ) for song in toplist_json['songlist']
+        ]
 
-        toplist_json = self._download_json(
-            jsonp_url, list_id, note='Retrieve toplist json',
-            errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
-
-        for song in toplist_json['l']:
-            s = song['s']
-            song_mid = s.split("|")[20]
-            entries.append(self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
-                song_mid))
-
-        list_name = self._html_search_regex(
-            r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
-            default=None)
-
-        return self.playlist_result(entries, list_id, list_name)
+        list_name = toplist_json['topinfo']['ListName']
+        list_description = toplist_json['topinfo']['info']
+        return self.playlist_result(entries, list_id, list_name, list_description)

From ed15e9ba02382fc7db22e6176068d2220c00a32e Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 17:32:06 +0800
Subject: [PATCH 0298/2145] [qqmusic] Remove unused import

---
 youtube_dl/extractor/qqmusic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 48f28ffe9..9943fcddb 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,7 +9,6 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
-    js_to_json,
 )
 from ..compat import compat_urllib_request
 

From 55e5841f14131ab61359535fdcc44e1564d555b8 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 17:41:29 +0800
Subject: [PATCH 0299/2145] [qqmusic] Extract additional formats (mp3-128,
 mp3-320)

---
 youtube_dl/extractor/qqmusic.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index b540033e2..4b4ef4993 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -19,10 +19,10 @@ class QQMusicIE(InfoExtractor):
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
-        'md5': 'bed90b6db2a7a7a7e11bc585f471f63a',
+        'md5': '9ce1c1c8445f561506d2e3cfb0255705',
         'info_dict': {
             'id': '004295Et37taLD',
-            'ext': 'm4a',
+            'ext': 'mp3',
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
@@ -30,6 +30,12 @@ class QQMusicIE(InfoExtractor):
         }
     }]
 
+    _FORMATS = {
+        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40},
+        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30},
+        'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
+    }
+
     # Reference: m_r_GetRUin() in top_player.js
     # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
     @staticmethod
@@ -69,11 +75,19 @@ class QQMusicIE(InfoExtractor):
             'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
             mid, note='Retrieve vkey', errnote='Unable to get vkey',
             transform_source=strip_jsonp)['key']
-        song_url = 'http://cc.stream.qqmusic.qq.com/C200%s.m4a?vkey=%s&guid=%s&fromtag=0' % (mid, vkey, guid)
+
+        formats = []
+        for k, sf in self._FORMATS.items():
+            formats.append({
+                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
+                       % (sf['prefix'], mid, sf['ext'], vkey, guid),
+                'format': k, 'format_id': k, 'preference': sf['preference']
+            })
+        self._sort_formats(formats)
 
         return {
             'id': mid,
-            'url': song_url,
+            'formats': formats,
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,

From b9258c61789388b49792ebdceb5d804217a36da5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:05:33 +0800
Subject: [PATCH 0300/2145] [YoutubeDL] Change how DashSegmentsFD is selected

---
 youtube_dl/downloader/__init__.py | 2 ++
 youtube_dl/downloader/http.py     | 4 ----
 youtube_dl/extractor/youtube.py   | 3 ++-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index f110830c4..1b618ab54 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -8,6 +8,7 @@ from .hls import NativeHlsFD
 from .http import HttpFD
 from .rtsp import RtspFD
 from .rtmp import RtmpFD
+from .dash import DashSegmentsFD
 
 from ..utils import (
     determine_protocol,
@@ -20,6 +21,7 @@ PROTOCOL_MAP = {
     'mms': RtspFD,
     'rtsp': RtspFD,
     'f4m': F4mFD,
+    'dash_segments': DashSegmentsFD,
 }
 
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index ceacb8522..b7f144af9 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -6,7 +6,6 @@ import socket
 import time
 
 from .common import FileDownloader
-from .dash import DashSegmentsFD
 from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
@@ -20,9 +19,6 @@ from ..utils import (
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
-        if info_dict.get('initialization_url') and list(filter(None, info_dict.get('segment_urls', []))):
-            return DashSegmentsFD(self.ydl, self.params).real_download(filename, info_dict)
-
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
         stream = None
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5d1297e0d..692d4d8db 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -819,7 +819,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if segment_list:
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')]
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'protocol': 'dash_segments',
                         })
                     try:
                         existing_format = next(

From 453a1617aac6e8000ed947cad7d88817c5740ede Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:12:05 +0800
Subject: [PATCH 0301/2145] [downloader/dash] Reorder imports

---
 youtube_dl/downloader/dash.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 18eca2c04..5f14658ba 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
-from .common import FileDownloader
-from ..compat import compat_urllib_request
 
 import re
 
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
 
 class DashSegmentsFD(FileDownloader):
     """

From 423d2be5f8c5e70d202ddfa63f3e5365e6afe823 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:27:29 +0800
Subject: [PATCH 0302/2145] [downloader/dash] Rename the protocol

'http_dash_segments' looks more like a protocol name than
'dash_segments'
---
 youtube_dl/downloader/__init__.py | 2 +-
 youtube_dl/extractor/youtube.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 1b618ab54..dccc59212 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -21,7 +21,7 @@ PROTOCOL_MAP = {
     'mms': RtspFD,
     'rtsp': RtspFD,
     'f4m': F4mFD,
-    'dash_segments': DashSegmentsFD,
+    'http_dash_segments': DashSegmentsFD,
 }
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 692d4d8db..6d288e848 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -820,7 +820,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
-                            'protocol': 'dash_segments',
+                            'protocol': 'http_dash_segments',
                         })
                     try:
                         existing_format = next(

From 56c837ccb75b639d362397095f33300229c4bd1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:34:48 +0600
Subject: [PATCH 0303/2145] [tnaflix] Fix typo

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 59af9aba0..bc51bae37 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -33,7 +33,7 @@ class TNAFlixIE(InfoExtractor):
         },
         {
             'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-            'matching_only': True,
+            'only_matching': True,
         }
     ]
 

From e52c0bd0eb853d4d242872e1d9ff5426a35dd30c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:37:05 +0600
Subject: [PATCH 0304/2145] [tnaflix] Modernize

---
 youtube_dl/extractor/tnaflix.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index bc51bae37..3e335d653 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -51,9 +51,8 @@ class TNAFlixIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        duration = self._html_search_meta('duration', webpage, 'duration', default=None)
-        if duration:
-            duration = parse_duration(duration[1:])
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', default=None))
 
         cfg_url = self._proto_relative_url(self._html_search_regex(
             self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')

From 3ce9bc712acd88df8499dd0982277c8f64b0d15a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:39:03 +0600
Subject: [PATCH 0305/2145] [empflix] Fix typo

---
 youtube_dl/extractor/empflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 9a5a8f4bb..4827022e0 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -26,6 +26,6 @@ class EMPFlixIE(TNAFlixIE):
         },
         {
             'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
-            'matching_only': True,
+            'only_matching': True,
         }
     ]

From 3d6388e34ea41d937f39e561b7731f1389971a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:42:37 +0600
Subject: [PATCH 0306/2145] [tnaflix] Fix relative URLs (empflix)

---
 youtube_dl/extractor/tnaflix.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 3e335d653..c282865b2 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -61,14 +61,15 @@ class TNAFlixIE(InfoExtractor):
             cfg_url, display_id, note='Downloading metadata',
             transform_source=fix_xml_ampersands)
 
-        thumbnail = cfg_xml.find('./startThumb').text
+        thumbnail = self._proto_relative_url(
+            cfg_xml.find('./startThumb').text, 'http:')
 
         formats = []
         for item in cfg_xml.findall('./quality/item'):
             video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
             format_id = item.find('res').text
             fmt = {
-                'url': video_url,
+                'url': self._proto_relative_url(video_url, 'http:'),
                 'format_id': format_id,
             }
             m = re.search(r'^(\d+)', format_id)

From 9d4f213f90d6024c7748f4defdc7b45f2351b0da Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 00:52:18 +0800
Subject: [PATCH 0307/2145] [qqmusic:toplist] List name and description are
 optional

---
 youtube_dl/extractor/qqmusic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 9943fcddb..f773332a8 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -222,6 +222,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             ) for song in toplist_json['songlist']
         ]
 
-        list_name = toplist_json['topinfo']['ListName']
-        list_description = toplist_json['topinfo']['info']
+        topinfo = toplist_json.get('topinfo', {})
+        list_name = topinfo.get('ListName')
+        list_description = topinfo.get('info')
         return self.playlist_result(entries, list_id, list_name, list_description)

From f5c78d118ba2d7e5e4a1ccd40c97fc1bf85a8dcf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Jun 2015 21:49:02 +0200
Subject: [PATCH 0308/2145] release 2015.06.04

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a421ae62b..d147b53fe 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -10,6 +10,7 @@
  - **56.com**
  - **5min**
  - **8tracks**
+ - **91porn**
  - **9gag**
  - **abc.net.au**
  - **Abc7News**
@@ -319,6 +320,7 @@
  - **Noco**
  - **Normalboots**
  - **NosVideo**
+ - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **novamov**: NovaMov
  - **Nowness**
  - **NowTV**
@@ -431,6 +433,8 @@
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
  - **Sohu**
+ - **soompi**
+ - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
  - **soundcloud:set**
@@ -505,6 +509,7 @@
  - **Trilulilu**
  - **TruTube**
  - **Tube8**
+ - **TubiTv**
  - **Tudou**
  - **Tumblr**
  - **TuneIn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 653710131..84224b7a7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.29'
+__version__ = '2015.06.04'

From 0e805e782bd05951ca3f420cf2a050e2ac3ae846 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Jun 2015 21:54:33 +0200
Subject: [PATCH 0309/2145] release 2015.06.04.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 84224b7a7..9cf84ff71 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.04'
+__version__ = '2015.06.04.1'

From 8b8cde21406b53f5aeb6586dab03a9d78d62e631 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 5 Jun 2015 06:04:26 +0800
Subject: [PATCH 0310/2145] [qqmusic] Set abr for mp3 formats

---
 youtube_dl/extractor/qqmusic.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 4b4ef4993..dc300e189 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -31,8 +31,8 @@ class QQMusicIE(InfoExtractor):
     }]
 
     _FORMATS = {
-        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40},
-        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30},
+        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40, 'abr': 320},
+        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30, 'abr': 128},
         'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
     }
 
@@ -77,11 +77,12 @@ class QQMusicIE(InfoExtractor):
             transform_source=strip_jsonp)['key']
 
         formats = []
-        for k, sf in self._FORMATS.items():
+        for k, f in self._FORMATS.items():
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (sf['prefix'], mid, sf['ext'], vkey, guid),
-                'format': k, 'format_id': k, 'preference': sf['preference']
+                       % (f['prefix'], mid, f['ext'], vkey, guid),
+                'format': k, 'format_id': k, 'preference': f['preference'],
+                'abr': f.get('abr')
             })
         self._sort_formats(formats)
 

From d31573fa37c8db7133492baf0a6be3ece643f8ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 22:55:29 +0800
Subject: [PATCH 0311/2145] [teamcoco] Handle incomplete m3u8 URLs (fixes
 #5798)

There are 2 TODOs. I don't know how to handle these cases correctly.
---
 youtube_dl/extractor/teamcoco.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index b2a4b1fc0..d1b7264b4 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -51,6 +51,17 @@ class TeamcocoIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # m3u8 downloads
             }
+        }, {
+            'url': 'http://teamcoco.com/video/full-episode-mon-6-1-joel-mchale-jake-tapper-and-musical-guest-courtney-barnett?playlist=x;eyJ0eXBlIjoidGFnIiwiaWQiOjl9',
+            'info_dict': {
+                'id': '89341',
+                'ext': 'mp4',
+                'title': 'Full Episode - Mon. 6/1 - Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
+                'description': 'Guests: Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 downloads
+            }
         }
     ]
     _VIDEO_ID_REGEXES = (
@@ -110,9 +121,23 @@ class TeamcocoIE(InfoExtractor):
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
             if determine_ext(filed['url']) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    filed['url'], video_id, ext='mp4'))
+                # compat_urllib_parse.urljoin does not work here
+                if filed['url'].startswith('/'):
+                    m3u8_url = 'http://ht.cdn.turner.com/tbs/big/teamcoco' + filed['url']
+                else:
+                    m3u8_url = filed['url']
+                m3u8_formats = self._extract_m3u8_formats(
+                    m3u8_url, video_id, ext='mp4')
+                for m3u8_format in m3u8_formats:
+                    if m3u8_format not in formats:
+                        formats.append(m3u8_format)
+            elif determine_ext(filed['url']) == 'f4m':
+                # TODO Correct f4m extraction
+                continue
             else:
+                if filed['url'].startswith('/mp4:protected/'):
+                    # TODO Correct extraction for these files
+                    continue
                 m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
                 if m_format is not None:
                     format_id = m_format.group(1)

From f00a650705a5e5b4f2b540ea8133a1752e63dd81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 23:16:34 +0800
Subject: [PATCH 0312/2145] [qqmusic] Rearrange codes

---
 youtube_dl/extractor/qqmusic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e24ddaefe..c903bee58 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -80,7 +80,9 @@ class QQMusicIE(InfoExtractor):
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
                        % (f['prefix'], mid, f['ext'], vkey, guid),
-                'format': k, 'format_id': k, 'preference': f['preference'],
+                'format': k,
+                'format_id': k,
+                'preference': f['preference'],
                 'abr': f.get('abr')
             })
         self._sort_formats(formats)

From e8ac61e840b5c02e406b910f5f0eed3d8b331969 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 23:19:25 +0800
Subject: [PATCH 0313/2145] [qqmusic] Use meaningful variable names

---
 youtube_dl/extractor/qqmusic.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c903bee58..bafa81c21 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -76,14 +76,14 @@ class QQMusicIE(InfoExtractor):
             transform_source=strip_jsonp)['key']
 
         formats = []
-        for k, f in self._FORMATS.items():
+        for format_id, details in self._FORMATS.items():
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (f['prefix'], mid, f['ext'], vkey, guid),
-                'format': k,
-                'format_id': k,
-                'preference': f['preference'],
-                'abr': f.get('abr')
+                       % (details['prefix'], mid, details['ext'], vkey, guid),
+                'format': format_id,
+                'format_id': format_id,
+                'preference': details['preference'],
+                'abr': details.get('abr'),
             })
         self._sort_formats(formats)
 

From dfe7dd9bdb45ec765c9b335c149e9913cf7e413f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 02:54:57 +0300
Subject: [PATCH 0314/2145] [izlesene] Unquote video URLs and simplify

---
 youtube_dl/extractor/izlesene.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 99a1361f8..753cb98ea 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -67,9 +68,9 @@ class IzleseneIE(InfoExtractor):
 
         uploader = self._html_search_regex(
             r"adduserUsername\s*=\s*'([^']+)';",
-            webpage, 'uploader', fatal=False, default='')
+            webpage, 'uploader', fatal=False)
         timestamp = parse_iso8601(self._html_search_meta(
-            'uploadDate', webpage, 'upload date', fatal=False))
+            'uploadDate', webpage, 'upload date'))
 
         duration = float_or_none(self._html_search_regex(
             r'"videoduration"\s*:\s*"([^"]+)"',
@@ -86,8 +87,7 @@ class IzleseneIE(InfoExtractor):
 
         # Might be empty for some videos.
         streams = self._html_search_regex(
-            r'"qualitylevel"\s*:\s*"([^"]+)"',
-            webpage, 'streams', fatal=False, default='')
+            r'"qualitylevel"\s*:\s*"([^"]+)"', webpage, 'streams', default='')
 
         formats = []
         if streams:
@@ -95,15 +95,15 @@ class IzleseneIE(InfoExtractor):
                 quality, url = re.search(r'\[(\w+)\](.+)', stream).groups()
                 formats.append({
                     'format_id': '%sp' % quality if quality else 'sd',
-                    'url': url,
+                    'url': compat_urllib_parse_unquote(url),
                     'ext': ext,
                 })
         else:
             stream_url = self._search_regex(
-                r'"streamurl"\s?:\s?"([^"]+)"', webpage, 'stream URL')
+                r'"streamurl"\s*:\s*"([^"]+)"', webpage, 'stream URL')
             formats.append({
                 'format_id': 'sd',
-                'url': stream_url,
+                'url': compat_urllib_parse_unquote(stream_url),
                 'ext': ext,
             })
 

From c33c547d66b20064f83932cdaa04823b17a96b70 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 02:57:12 +0300
Subject: [PATCH 0315/2145] [izlesene] Avoid timestamp differences in tests due
 to DST

---
 youtube_dl/extractor/izlesene.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 753cb98ea..bc226fa67 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -31,7 +31,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'pelikzzle',
-                'timestamp': 1404302298,
+                'timestamp': int,
                 'upload_date': '20140702',
                 'duration': 95.395,
                 'age_limit': 0,
@@ -47,7 +47,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'Tarkan Dortmund 2006 Konseri',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'parlayankiz',
-                'timestamp': 1163322193,
+                'timestamp': int,
                 'upload_date': '20061112',
                 'duration': 253.666,
                 'age_limit': 0,

From 54eb81a087516e9d040bc1ad274c0a64b51dd1d1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 03:11:43 +0300
Subject: [PATCH 0316/2145] [pornovoisines] Improve average_rating extraction
 and update test case

---
 youtube_dl/extractor/pornovoisines.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index 9688ed948..eba4dfbb3 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -34,7 +34,7 @@ class PornoVoisinesIE(InfoExtractor):
             'duration': 120,
             'view_count': int,
             'average_rating': float,
-            'categories': ['Débutante', 'Scénario', 'Sodomie'],
+            'categories': ['Débutantes', 'Scénario', 'Sodomie'],
             'age_limit': 18,
         }
     }
@@ -71,7 +71,7 @@ class PornoVoisinesIE(InfoExtractor):
         view_count = int_or_none(self._search_regex(
             r'(\d+) vues', webpage, 'view count', fatal=False))
         average_rating = self._search_regex(
-            r'Note : (\d+,\d+)', webpage, 'average rating', fatal=False)
+            r'Note\s*:\s*(\d+(?:,\d+)?)', webpage, 'average rating', fatal=False)
         if average_rating:
             average_rating = float_or_none(average_rating.replace(',', '.'))
 

From 3d8e9573a470594df6fa471dc33c4c4b938b668a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jun 2015 06:25:37 +0600
Subject: [PATCH 0317/2145] [youtube:channel] Improve channel id extraction
 (#5904)

---
 youtube_dl/extractor/youtube.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aacb999ce..419f7b019 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1406,10 +1406,12 @@ class YoutubeChannelIE(InfoExtractor):
         channel_page = self._download_webpage(
             url + '?view=57', channel_id,
             'Downloading channel page', fatal=False)
-        channel_playlist_id = self._search_regex(
-            [r'<meta itemprop="channelId" content="([^"]+)">',
-             r'data-channel-external-id="([^"]+)"'],
-            channel_page, 'channel id', default=None)
+        channel_playlist_id = self._html_search_meta(
+            'channelId', channel_page, 'channel id', default=None)
+        if not channel_playlist_id:
+            channel_playlist_id = self._search_regex(
+                r'data-channel-external-id="([^"]+)"',
+                channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
             return self.url_result(

From 223544552fcfec0c5c6a83326520c614e4489cbb Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sat, 9 May 2015 03:53:43 +0300
Subject: [PATCH 0318/2145] [Ruutu] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ruutu.py    | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/ruutu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8ec0c1032..860023d14 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -448,6 +448,7 @@ from .rutube import (
     RutubePersonIE,
 )
 from .rutv import RUTVIE
+from .ruutu import RuutuIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
new file mode 100644
index 000000000..e346434f9
--- /dev/null
+++ b/youtube_dl/extractor/ruutu.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
+import re
+
+
+class RuutuIE(InfoExtractor):
+    _VALID_URL = r'http://(www\.)?ruutu\.fi/ohjelmat/(?:[^/]+/)?(?P<id>.*)$'
+    _TESTS = [
+        {
+            'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+            'md5': 'ab2093f39be1ca8581963451b3c0234f',
+            'info_dict': {
+                'id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+                'ext': 'mp4',
+                'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
+                'description': 'Toinen toistaan huikeampia ohjelmaideoita ja täysin päätöntä sekoilua? No sitä juuri nimenomaan. Metro Helsingin Iltapäivän vieraaksi saapui Tuomas Kauhanen ja he Petra Kalliomaan kanssa keskustelivat hieman ennen lähetyksen alkua, mutta kamerat olivatkin jo päällä.',
+            },
+            'params': {
+                'format': 'http-1000',
+            }
+        },
+        {
+            'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
+            'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
+            'info_dict': {
+                'id': 'superpesis-katso-koko-kausi-ruudussa',
+                'ext': 'mp4',
+                'title': 'Superpesis: katso koko kausi Ruudussa',
+                'description': 'Huippujännittävän Superpesiksen suoria ottelulähetyksiä seurataan Ruudussa kauden alusta viimeiseen finaaliin asti. Katso lisätiedot osoitteesta ruutu.fi/superpesis.',
+            },
+            'params': {
+                'format': 'http-1000',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id)
+        media_id = self._html_search_regex(r'data-media-id="(\d+)"', webpage, 'media_id')
+        media_json = self._parse_json(self._search_regex(r'jQuery.extend\([^,]+, (.*)\);', webpage, 'media_data'), video_id)
+        xml_url = media_json['ruutuplayer']['xmlUrl'].replace('{ID}', media_id)
+        media_xml = self._download_xml(xml_url, media_id)
+
+        formats = []
+        parsed_urls = []
+        for fmt in media_xml.findall('.//Clip//'):
+            url = fmt.text
+            if not fmt.tag.endswith('File') or url in parsed_urls or \
+                    'NOT_USED' in url:
+                continue
+
+            if url.endswith('m3u8'):
+                formats.extend(self._extract_m3u8_formats(url, media_id, m3u8_id='hls'))
+                parsed_urls.append(url)
+            elif url.endswith('f4m'):
+                formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
+                parsed_urls.append(url)
+            else:
+                proto = compat_urllib_parse_urlparse(url).scheme
+                width_str, height_str = fmt.get('resolution').split('x')
+                tbr = int(fmt.get('bitrate', 0))
+                formats.append({
+                    'format_id': '%s-%d' % (proto, tbr),
+                    'url': url,
+                    'width': int(width_str),
+                    'height': int(height_str),
+                    'tbr': tbr,
+                    'ext': url.rsplit('.', 1)[-1],
+                    'live': True,
+                    'protocol': proto,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int(media_xml.find('.//Runtime').text),
+            'age_limit': int(media_xml.find('.//AgeLimit').text),
+        }

From a9e58ecd3fdfa93cdc8a7f9fc852dbbd0814d6a4 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 13:56:46 +0300
Subject: [PATCH 0319/2145] [turbo] Improve description extraction

`og:description` is empty for some videos.
---
 youtube_dl/extractor/turbo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/turbo.py b/youtube_dl/extractor/turbo.py
index 29703a8a9..7ae63a499 100644
--- a/youtube_dl/extractor/turbo.py
+++ b/youtube_dl/extractor/turbo.py
@@ -23,7 +23,7 @@ class TurboIE(InfoExtractor):
             'ext': 'mp4',
             'duration': 3715,
             'title': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
-            'description': 'Retrouvez dans cette rubrique toutes les vidéos de l\'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
+            'description': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -42,7 +42,7 @@ class TurboIE(InfoExtractor):
         title = xpath_text(item, './title', 'title')
         duration = int_or_none(xpath_text(item, './durate', 'duration'))
         thumbnail = xpath_text(item, './visuel_clip', 'thumbnail')
-        description = self._og_search_description(webpage)
+        description = self._html_search_meta('description', webpage)
 
         formats = []
         get_quality = qualities(['3g', 'sd', 'hq'])

From 05aa9c82d90644af406519e5e25fefb0884d504e Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 13:58:20 +0300
Subject: [PATCH 0320/2145] [sunporno] Fix view_count extraction

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 854d01bee..e527aa971 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -44,7 +44,7 @@ class SunPornoIE(InfoExtractor):
             webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'class="views">\s*(\d+)\s*<',
+            r'class="views">(?:<noscript>)?\s*(\d+)\s*<',
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'(\d+)</b> Comments?',

From 4da31bd56629054497634d041035e4bd6fcfacbb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 Jun 2015 22:22:26 +0800
Subject: [PATCH 0321/2145] [youtube] Fix a FutureWarning from
 xml.etree.ElementTree

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d288e848..2424ac2c0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -816,7 +816,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
-                    if segment_list:
+                    if len(segment_list):
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],

From f1da861018924e6f442ffedd9a5682055c79aea6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:37:29 +0800
Subject: [PATCH 0322/2145] [iqiyi] PEP8

---
 youtube_dl/extractor/iqiyi.py | 56 +++++++++++++++++------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 747f3f902..597441baf 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,19 +16,20 @@ import random
 import zlib
 import hashlib
 
+
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
 
     _TEST = {
-            'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-            'md5': '2cb594dc2781e6c941a110d8f358118b',
-            'info_dict': {
-                'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
-                'title': '美国德州空中惊现奇异云团 酷似UFO',
-                'ext': 'f4v',
-            }
+        'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
+        'md5': '2cb594dc2781e6c941a110d8f358118b',
+        'info_dict': {
+            'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+            'title': '美国德州空中惊现奇异云团 酷似UFO',
+            'ext': 'f4v',
+        }
     }
 
     def construct_video_urls(self, data, video_id, _uuid, bid):
@@ -46,7 +47,7 @@ class IqiyiIE(InfoExtractor):
             c = len(b)
             s = ''
             for i in range(c - 1, -1, -1):
-                a = do_xor(int(b[c-i-1], 16), i)
+                a = do_xor(int(b[c - i - 1], 16), i)
                 s += chr(a)
             return s[::-1]
 
@@ -54,15 +55,14 @@ class IqiyiIE(InfoExtractor):
             mg = ')(*&^flash@#$%a'
             tm = self._download_json(
                 'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
-            t = str(int(math.floor(int(tm)/(600.0))))
-            return hashlib.md5(
-                 (t+mg+x).encode('utf8')).hexdigest()
+            t = str(int(math.floor(int(tm) / (600.0))))
+            return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
         # get accept format
         # getting all format will spend minutes for a big video.
         if bid == 'best':
-            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs'] \
-                   if 0 < int(i['bid']) <= 10]
+            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs']
+                    if 0 < int(i['bid']) <= 10]
             bid = str(max(bids))
 
         video_urls_dict = {}
@@ -117,24 +117,24 @@ class IqiyiIE(InfoExtractor):
 
     def get_format(self, bid):
         _dict = {
-            '1'  : 'h6',
-            '2'  : 'h5',
-            '3'  : 'h4',
-            '4'  : 'h3',
-            '5'  : 'h2',
-            '10' : 'h1'
+            '1': 'h6',
+            '2': 'h5',
+            '3': 'h4',
+            '4': 'h3',
+            '5': 'h2',
+            '10': 'h1'
         }
         return _dict.get(str(bid), None)
 
     def get_bid(self, format_id):
         _dict = {
-            'h6'   : '1',
-            'h5'   : '2',
-            'h4'   : '3',
-            'h3'   : '4',
-            'h2'   : '5',
-            'h1'   : '10',
-            'best' : 'best'
+            'h6': '1',
+            'h5': '2',
+            'h4': '3',
+            'h3': '4',
+            'h2': '5',
+            'h1': '10',
+            'best': 'best'
         }
         return _dict.get(format_id, None)
 
@@ -207,7 +207,7 @@ class IqiyiIE(InfoExtractor):
         for format_id in video_urls_dict:
             video_urls = video_urls_dict[format_id]
             for i, video_url_info in enumerate(video_urls):
-                if len(entries) < i+1:
+                if len(entries) < i + 1:
                     entries.append({'formats': []})
                 entries[i]['formats'].append(
                     {
@@ -222,7 +222,7 @@ class IqiyiIE(InfoExtractor):
             self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
-                    'id': '_part%d' % (i+1),
+                    'id': '_part%d' % (i + 1),
                     'title': title,
                 }
             )

From 7012620e2b9355d25ddfc855fc5990af938f04d8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:44:54 +0800
Subject: [PATCH 0323/2145] [iqiyi] Remove format selection codes

---
 youtube_dl/extractor/iqiyi.py | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 597441baf..5645fb6ee 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -32,7 +32,7 @@ class IqiyiIE(InfoExtractor):
         }
     }
 
-    def construct_video_urls(self, data, video_id, _uuid, bid):
+    def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -58,13 +58,6 @@ class IqiyiIE(InfoExtractor):
             t = str(int(math.floor(int(tm) / (600.0))))
             return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
-        # get accept format
-        # getting all format will spend minutes for a big video.
-        if bid == 'best':
-            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs']
-                    if 0 < int(i['bid']) <= 10]
-            bid = str(max(bids))
-
         video_urls_dict = {}
         for i in data['vp']['tkl'][0]['vs']:
             if 0 < int(i['bid']) <= 10:
@@ -80,12 +73,6 @@ class IqiyiIE(InfoExtractor):
                 if t.endswith('mp4'):
                     video_urls_info = i['flvs']
 
-            if int(i['bid']) != int(bid):  # ignore missing match format
-                video_urls.extend(
-                    [('http://example.com/v.flv', ii['b']) for ii in video_urls_info])
-                video_urls_dict[format_id] = video_urls
-                continue
-
             for ii in video_urls_info:
                 vl = ii['l']
                 if not vl.startswith('/'):
@@ -193,14 +180,9 @@ class IqiyiIE(InfoExtractor):
 
         title = data['vi']['vn']
 
-        format = self._downloader.params.get('format', None)
-        bid = self.get_bid(format) if format else 'best'
-        if not bid:
-            raise ExtractorError('Can\'t get format.')
-
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(
-            data, video_id, _uuid, bid)
+            data, video_id, _uuid)
 
         # construct info
         entries = []

From 29e7e0781b1b8e276c28a079bc5b18e1b0db2d5e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:56:08 +0800
Subject: [PATCH 0324/2145] [iqiyi] Simplify and improve regex patterns

See the comments in #5849
---
 youtube_dl/extractor/iqiyi.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 5645fb6ee..18a7587a2 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -161,12 +161,11 @@ class IqiyiIE(InfoExtractor):
         webpage = self._download_webpage(
             url, 'temp_id', note='download video page')
         tvid = self._search_regex(
-            r'tvId ?= ?(\'|\")(?P<tvid>\d+)', webpage, 'tvid', flags=re.I, group='tvid')
+            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(
-            r'videoId ?= ?(\'|\")(?P<video_id>[a-z\d]+)',
-            webpage, 'video_id', flags=re.I, group='video_id')
+            r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
         swf_url = self._search_regex(
-            r'(?P<swf>http://.+?MainPlayer.+?\.swf)', webpage, 'swf')
+            r'(http://.+?MainPlayer.+?\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
         enc_key = self.get_enc_key(swf_url, video_id)

From aacda28b28c1804866d634c5c5086b3d53cb2b2f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:32:03 +0800
Subject: [PATCH 0325/2145] [iqiyi] Give error message for assertion failures

---
 youtube_dl/extractor/iqiyi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 18a7587a2..dc35c3380 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -171,7 +171,10 @@ class IqiyiIE(InfoExtractor):
         enc_key = self.get_enc_key(swf_url, video_id)
 
         raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
-        assert raw_data['code'] == 'A000000'
+
+        if raw_data['code'] != 'A000000':
+            raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
+
         if not raw_data['data']['vp']['tkl']:
             raise ExtractorError('No support iQiqy VIP video')
 

From 958d0b659b80d4493d045d4da82074ed68ed6c4e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:35:09 +0800
Subject: [PATCH 0326/2145] [iqiyi] Reorder imports

---
 youtube_dl/extractor/iqiyi.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index dc35c3380..36029361a 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -1,20 +1,17 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-
-from ..compat import compat_urllib_parse
-
-from ..utils import ExtractorError
-
+import hashlib
+import math
+import random
 import re
 import time
 import uuid
-import math
-import random
 import zlib
-import hashlib
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import ExtractorError
 
 
 class IqiyiIE(InfoExtractor):

From ffba4edb067238b593b98c71f4293e9b60ba95ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:52:51 +0800
Subject: [PATCH 0327/2145] [iqiyi] Improve some variable names and add
 download notes

---
 youtube_dl/extractor/iqiyi.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 36029361a..c17e1fde4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -48,35 +48,37 @@ class IqiyiIE(InfoExtractor):
                 s += chr(a)
             return s[::-1]
 
-        def get_path_key(x):
+        def get_path_key(x, format_id, segment_index):
             mg = ')(*&^flash@#$%a'
             tm = self._download_json(
-                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
+                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id,
+                note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
+            )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
             return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
         video_urls_dict = {}
-        for i in data['vp']['tkl'][0]['vs']:
-            if 0 < int(i['bid']) <= 10:
-                format_id = self.get_format(i['bid'])
+        for format_item in data['vp']['tkl'][0]['vs']:
+            if 0 < int(format_item['bid']) <= 10:
+                format_id = self.get_format(format_item['bid'])
             else:
                 continue
 
             video_urls = []
 
-            video_urls_info = i['fs']
-            if not i['fs'][0]['l'].startswith('/'):
-                t = get_encode_code(i['fs'][0]['l'])
+            video_urls_info = format_item['fs']
+            if not format_item['fs'][0]['l'].startswith('/'):
+                t = get_encode_code(format_item['fs'][0]['l'])
                 if t.endswith('mp4'):
-                    video_urls_info = i['flvs']
+                    video_urls_info = format_item['flvs']
 
-            for ii in video_urls_info:
-                vl = ii['l']
+            for segment_index, segment in enumerate(video_urls_info):
+                vl = segment['l']
                 if not vl.startswith('/'):
                     vl = get_encode_code(vl)
                 key = get_path_key(
-                    vl.split('/')[-1].split('.')[0])
-                filesize = ii['b']
+                    vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                filesize = segment['b']
                 base_url = data['vp']['du'].split('/')
                 base_url.insert(-1, key)
                 base_url = '/'.join(base_url)
@@ -91,7 +93,9 @@ class IqiyiIE(InfoExtractor):
                 }
                 api_video_url = base_url + vl + '?' + \
                     compat_urllib_parse.urlencode(param)
-                js = self._download_json(api_video_url, video_id)
+                js = self._download_json(
+                    api_video_url, video_id,
+                    note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
                 video_url = js['l']
                 video_urls.append(
                     (video_url, filesize))

From c4ee87022bd18863fc3f22f80064453e272d956f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:57:05 +0800
Subject: [PATCH 0328/2145] [iqiyi] Change id for multipart videos

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c17e1fde4..840cc9a4d 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -207,7 +207,7 @@ class IqiyiIE(InfoExtractor):
             self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
-                    'id': '_part%d' % (i + 1),
+                    'id': '%s_part%d' % (video_id, i + 1),
                     'title': title,
                 }
             )

From 99481135907b5fa3558d4f176fd02acbdafccdb6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:09:33 +0800
Subject: [PATCH 0329/2145] [iqiyi] Add a multipart test case

---
 youtube_dl/extractor/iqiyi.py | 67 +++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 840cc9a4d..d73687d88 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -19,7 +19,7 @@ class IqiyiIE(InfoExtractor):
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
         'md5': '2cb594dc2781e6c941a110d8f358118b',
         'info_dict': {
@@ -27,7 +27,70 @@ class IqiyiIE(InfoExtractor):
             'title': '美国德州空中惊现奇异云团 酷似UFO',
             'ext': 'f4v',
         }
-    }
+    }, {
+        'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
+        'info_dict': {
+            'id': 'e3f585b550a280af23c98b6cb2be19fb',
+            'title': '名侦探柯南第752集',
+        },
+        'playlist': [{
+            'md5': '7e49376fecaffa115d951634917fe105',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '41b75ba13bb7ac0e411131f92bc4f6ca',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '0cee1dd0a3d46a83e71e2badeae2aab0',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '4f8ad72373b0c491b582e7c196b0b1f9',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': 'd89ad028bcfad282918e8098e811711d',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '9cb1e5c95da25dff0660c32ae50903b7',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '155116e0ff1867bbc9b98df294faabc9',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '53f5db77622ae14fa493ed2a278a082b',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }],
+    }]
 
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):

From 865ab62f43eb94a9f4f757a464df147e983cb439 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:13:22 +0800
Subject: [PATCH 0330/2145] [iqiyi] Make _VALID_URL more accurate

v_* urls are individual videos, while a_* urls are playlists, which are
not supported yet.
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d73687d88..f0d423331 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -17,7 +17,7 @@ from ..utils import ExtractorError
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From 08bb8ef2011d795948d8e89478bf3afe4b99405f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:25:00 +0800
Subject: [PATCH 0331/2145] [iqiyi] Unify get_format() and get_bid()

---
 youtube_dl/extractor/iqiyi.py | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f0d423331..122f33692 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -92,6 +92,15 @@ class IqiyiIE(InfoExtractor):
         }],
     }]
 
+    _FORMATS_MAP = [
+        ('1', 'h6'),
+        ('2', 'h5'),
+        ('3', 'h4'),
+        ('4', 'h3'),
+        ('5', 'h2'),
+        ('10', 'h1'),
+    ]
+
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
@@ -167,27 +176,12 @@ class IqiyiIE(InfoExtractor):
         return video_urls_dict
 
     def get_format(self, bid):
-        _dict = {
-            '1': 'h6',
-            '2': 'h5',
-            '3': 'h4',
-            '4': 'h3',
-            '5': 'h2',
-            '10': 'h1'
-        }
-        return _dict.get(str(bid), None)
+        matched_format_ids = [_format_id for _bid, _format_id in self._FORMATS_MAP if _bid == str(bid)]
+        return matched_format_ids[0] if len(matched_format_ids) else None
 
     def get_bid(self, format_id):
-        _dict = {
-            'h6': '1',
-            'h5': '2',
-            'h4': '3',
-            'h3': '4',
-            'h2': '5',
-            'h1': '10',
-            'best': 'best'
-        }
-        return _dict.get(format_id, None)
+        matched_bids = [_bid for _bid, _format_id in self._FORMATS_MAP if _format_id == format_id]
+        return matched_bids[0] if len(matched_bids) else None
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))

From 9c5f685ef14a8b44d17b897ba8ae2da051011c35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:39:03 +0800
Subject: [PATCH 0332/2145] [iqiyi] Improve regex pattern again

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 122f33692..15481b84b 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -223,7 +223,7 @@ class IqiyiIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
         swf_url = self._search_regex(
-            r'(http://.+?MainPlayer.+?\.swf)', webpage, 'swf player URL')
+            r'(http://[^\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
         enc_key = self.get_enc_key(swf_url, video_id)

From b5a3c7f10927c9d55f6fdad5f5c002e02338642e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:47:36 +0800
Subject: [PATCH 0333/2145] [iqiyi] Cache encryption keys

---
 youtube_dl/extractor/iqiyi.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 15481b84b..9106dd074 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
+import os.path
 import random
 import re
 import time
@@ -11,7 +12,10 @@ import zlib
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    url_basename,
+)
 
 
 class IqiyiIE(InfoExtractor):
@@ -207,12 +211,20 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
+        filename, _ = os.path.splitext(url_basename(swf_url))
+        enc_key_json = self._downloader.cache.load('iqiyi-enc-key', filename)
+        if enc_key_json is not None:
+            return enc_key_json[0]
+
         req = self._request_webpage(
             swf_url, video_id, note='download swf content')
         cn = req.read()
         cn = zlib.decompress(cn[8:])
         pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
         enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
+
+        self._downloader.cache.store('iqiyi-enc-key', filename, [enc_key])
+
         return enc_key
 
     def _real_extract(self, url):

From d00735a0c5aabd38b37bfea76a93ae8c47a8d419 Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sat, 6 Jun 2015 23:01:23 +0300
Subject: [PATCH 0334/2145] [ruutu] Don't use fallback for DASH and other
 non-HTTP urls

---
 youtube_dl/extractor/ruutu.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index e346434f9..59e0b12fd 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -62,6 +62,8 @@ class RuutuIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
                 parsed_urls.append(url)
             else:
+                if not fmt.tag.startswith('HTTP'):
+                    continue
                 proto = compat_urllib_parse_urlparse(url).scheme
                 width_str, height_str = fmt.get('resolution').split('x')
                 tbr = int(fmt.get('bitrate', 0))

From de390ea0771a0e35c0c2970bc00f5fa2dd9d3eac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 7 Jun 2015 00:19:45 +0200
Subject: [PATCH 0335/2145] update: Use https for getting the version info
 (fixes #5909)

---
 youtube_dl/update.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index de3169eef..fc7ac8305 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -50,7 +50,7 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
+    UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

From 9414338a48ca815fd666aad496ebabd6d0c76e5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jun 2015 05:37:29 +0600
Subject: [PATCH 0336/2145] [ruutu] Improve, make more robust and fix python
 2.6 support

---
 youtube_dl/extractor/ruutu.py | 129 +++++++++++++++++++++-------------
 1 file changed, 79 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 59e0b12fd..4e22628d0 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -3,88 +3,117 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
-import re
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    xpath_text,
+)
 
 
 class RuutuIE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?ruutu\.fi/ohjelmat/(?:[^/]+/)?(?P<id>.*)$'
+    _VALID_URL = r'http://(?:www\.)?ruutu\.fi/ohjelmat/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
     _TESTS = [
         {
             'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
             'md5': 'ab2093f39be1ca8581963451b3c0234f',
             'info_dict': {
-                'id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+                'id': '2058907',
+                'display_id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
                 'ext': 'mp4',
                 'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
-                'description': 'Toinen toistaan huikeampia ohjelmaideoita ja täysin päätöntä sekoilua? No sitä juuri nimenomaan. Metro Helsingin Iltapäivän vieraaksi saapui Tuomas Kauhanen ja he Petra Kalliomaan kanssa keskustelivat hieman ennen lähetyksen alkua, mutta kamerat olivatkin jo päällä.',
+                'description': 'md5:cfc6ccf0e57a814360df464a91ff67d6',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 114,
+                'age_limit': 0,
             },
-            'params': {
-                'format': 'http-1000',
-            }
         },
         {
             'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
-                'id': 'superpesis-katso-koko-kausi-ruudussa',
+                'id': '2057306',
+                'display_id': 'superpesis-katso-koko-kausi-ruudussa',
                 'ext': 'mp4',
                 'title': 'Superpesis: katso koko kausi Ruudussa',
-                'description': 'Huippujännittävän Superpesiksen suoria ottelulähetyksiä seurataan Ruudussa kauden alusta viimeiseen finaaliin asti. Katso lisätiedot osoitteesta ruutu.fi/superpesis.',
+                'description': 'md5:44c44a99fdbe5b380ab74ebd75f0af77',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 40,
+                'age_limit': 0,
             },
-            'params': {
-                'format': 'http-1000',
-            }
         },
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        display_id = self._match_id(url)
 
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        media_id = self._html_search_regex(r'data-media-id="(\d+)"', webpage, 'media_id')
-        media_json = self._parse_json(self._search_regex(r'jQuery.extend\([^,]+, (.*)\);', webpage, 'media_data'), video_id)
-        xml_url = media_json['ruutuplayer']['xmlUrl'].replace('{ID}', media_id)
-        media_xml = self._download_xml(xml_url, media_id)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'data-media-id="(\d+)"', webpage, 'media id')
+
+        video_xml_url = None
+
+        media_data = self._search_regex(
+            r'jQuery\.extend\([^,]+,\s*(.+?)\);', webpage,
+            'media data', default=None)
+        if media_data:
+            media_json = self._parse_json(media_data, display_id, fatal=False)
+            if media_json:
+                xml_url = media_json.get('ruutuplayer', {}).get('xmlUrl')
+                if xml_url:
+                    video_xml_url = xml_url.replace('{ID}', video_id)
+
+        if not video_xml_url:
+            video_xml_url = 'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id
+
+        video_xml = self._download_xml(video_xml_url, video_id)
 
         formats = []
-        parsed_urls = []
-        for fmt in media_xml.findall('.//Clip//'):
-            url = fmt.text
-            if not fmt.tag.endswith('File') or url in parsed_urls or \
-                    'NOT_USED' in url:
-                continue
+        processed_urls = []
 
-            if url.endswith('m3u8'):
-                formats.extend(self._extract_m3u8_formats(url, media_id, m3u8_id='hls'))
-                parsed_urls.append(url)
-            elif url.endswith('f4m'):
-                formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
-                parsed_urls.append(url)
-            else:
-                if not fmt.tag.startswith('HTTP'):
-                    continue
-                proto = compat_urllib_parse_urlparse(url).scheme
-                width_str, height_str = fmt.get('resolution').split('x')
-                tbr = int(fmt.get('bitrate', 0))
-                formats.append({
-                    'format_id': '%s-%d' % (proto, tbr),
-                    'url': url,
-                    'width': int(width_str),
-                    'height': int(height_str),
-                    'tbr': tbr,
-                    'ext': url.rsplit('.', 1)[-1],
-                    'live': True,
-                    'protocol': proto,
-                })
+        def extract_formats(node):
+            for child in node:
+                if child.tag.endswith('Files'):
+                    extract_formats(child)
+                elif child.tag.endswith('File'):
+                    video_url = child.text
+                    if not video_url or video_url in processed_urls or 'NOT_USED' in video_url:
+                        return
+                    processed_urls.append(video_url)
+                    ext = determine_ext(video_url)
+                    if ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls'))
+                    elif ext == 'f4m':
+                        formats.extend(self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds'))
+                    else:
+                        proto = compat_urllib_parse_urlparse(video_url).scheme
+                        if not child.tag.startswith('HTTP') and proto != 'rtmp':
+                            continue
+                        preference = -1 if proto == 'rtmp' else 1
+                        label = child.get('label')
+                        tbr = int_or_none(child.get('bitrate'))
+                        width, height = [int_or_none(x) for x in child.get('resolution', '').split('x')]
+                        formats.append({
+                            'format_id': '%s-%s' % (proto, label if label else tbr),
+                            'url': video_url,
+                            'width': width,
+                            'height': height,
+                            'tbr': tbr,
+                            'preference': preference,
+                        })
+
+        extract_formats(video_xml.find('./Clip'))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': self._og_search_title(webpage),
-            'formats': formats,
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': int(media_xml.find('.//Runtime').text),
-            'age_limit': int(media_xml.find('.//AgeLimit').text),
+            'duration': int_or_none(xpath_text(video_xml, './/Runtime', 'duration')),
+            'age_limit': int_or_none(xpath_text(video_xml, './/AgeLimit', 'age limit')),
+            'formats': formats,
         }

From 9836cfb8d682c91036ce417fa31200673b52115b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jun 2015 08:12:21 +0600
Subject: [PATCH 0337/2145] [options] Clarify `--list-extractors` (Closes
 #5916)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5a2315bd9..689fa7595 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -145,7 +145,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
-        help='List all supported extractors and the URLs they would handle')
+        help='List all supported extractors')
     general.add_option(
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,

From b26733ba7f376f8c9285ac7928534286622bbc7c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 15:29:17 +0800
Subject: [PATCH 0338/2145] [brightcove] Allow single quotes in Brightcove URLs
 (fixes #5901)

---
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/generic.py    | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4f60d5366..c1d4320e1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -172,7 +172,7 @@ class BrightcoveIE(InfoExtractor):
         """Return a list of all Brightcove URLs from the webpage """
 
         url_m = re.search(
-            r'<meta\s+property="og:video"\s+content="(https?://(?:secure|c)\.brightcove.com/[^"]+)"',
+            r'<meta\s+property=[\'"]og:video[\'"]\s+content=[\'"](https?://(?:secure|c)\.brightcove.com/[^\'"]+)[\'"]',
             webpage)
         if url_m:
             url = unescapeHTML(url_m.group(1))
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 96ca398de..759691365 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -789,6 +789,18 @@ class GenericIE(InfoExtractor):
                 # rtmpe downloads
                 'skip_download': True,
             }
+        },
+        # Brightcove URL in single quotes
+        {
+            'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
+            'md5': '4ae374f1f8b91c889c4b9203c8c752af',
+            'info_dict': {
+                'id': '4255764656001',
+                'ext': 'mp4',
+                'title': 'SN Presents: Russell Martin, World Citizen',
+                'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
+                'uploader': 'Rogers Sportsnet',
+            },
         }
     ]
 

From 621ed9f5f4d9d82659272ebe01e740e9196fad61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:33:22 +0800
Subject: [PATCH 0339/2145] [common] Add note and errnote field for
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cecf917ff..49e4dc710 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,7 @@ class InfoExtractor(object):
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
-                              m3u8_id=None):
+                              m3u8_id=None, note=None, errnote=None):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -865,8 +865,8 @@ class InfoExtractor(object):
 
         m3u8_doc = self._download_webpage(
             m3u8_url, video_id,
-            note='Downloading m3u8 information',
-            errnote='Failed to download m3u8 information')
+            note=note or 'Downloading m3u8 information',
+            errnote=errnote or 'Failed to download m3u8 information')
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 65ba8b23f471b96e6f937f2754c729e22bf2cf0a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:34:19 +0800
Subject: [PATCH 0340/2145] [discovery] Rewrite DiscoveryIE (fixes #5898)

Discovery.com now uses a completely different approach for serving
videos. At least in both test cases brightcove are involved. However,
AMF support is necessary for these brightcove videos. As a result, I
try to extract videos from the info page ('?flat=1'). The downloaded
file can be different from the one in browsers.
---
 youtube_dl/extractor/discovery.py | 52 ++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index d3e667528..d6723ecf2 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -2,19 +2,19 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    parse_duration,
     parse_iso8601,
-    int_or_none,
 )
+from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
     _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
-        'md5': '3c69d77d9b0d82bfd5e5932a60f26504',
         'info_dict': {
-            'id': 'mission-impossible-outtakes',
-            'ext': 'flv',
+            'id': '20769',
+            'ext': 'mp4',
             'title': 'Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                             ' each other -- to the point of confusing Jamie\'s dog -- and '
@@ -24,22 +24,36 @@ class DiscoveryIE(InfoExtractor):
             'timestamp': 1303099200,
             'upload_date': '20110418',
         },
-    }
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        }
+    }, {
+        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mythbusters-the-simpsons',
+        'info_dict': {
+            'id': 'mythbusters-the-simpsons',
+            'title': 'MythBusters: The Simpsons',
+        },
+        'playlist_count': 9,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        info = self._download_json(url + '?flat=1', video_id)
 
-        info = self._parse_json(self._search_regex(
-            r'(?s)<script type="application/ld\+json">(.*?)</script>',
-            webpage, 'video info'), video_id)
+        video_title = info.get('playlist_title') or info.get('video_title')
 
-        return {
-            'id': video_id,
-            'title': info['name'],
-            'url': info['contentURL'],
-            'description': info.get('description'),
-            'thumbnail': info.get('thumbnailUrl'),
-            'timestamp': parse_iso8601(info.get('uploadDate')),
-            'duration': int_or_none(info.get('duration')),
-        }
+        entries = [{
+            'id': compat_str(video_info['id']),
+            'formats': self._extract_m3u8_formats(
+                video_info['src'], video_id, ext='mp4',
+                note='Download m3u8 information for video %d' % (idx + 1)),
+            'title': video_info['title'],
+            'description': video_info.get('description'),
+            'duration': parse_duration(video_info.get('video_length')),
+            'webpage_url': video_info.get('href'),
+            'thumbnail': video_info.get('thumbnailURL'),
+            'alt_title': video_info.get('secondary_title'),
+            'timestamp': parse_iso8601(video_info.get('publishedDate')),
+        } for idx, video_info in enumerate(info['playlist'])]
+
+        return self.playlist_result(entries, video_id, video_title)

From 68477c3dab97733eb7a2feb8fcc90f648c29c2b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:38:39 +0800
Subject: [PATCH 0341/2145] [tlc] Fix test failure due to DiscoveryIE changes

---
 youtube_dl/extractor/tlc.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 9f9e388c5..13263614c 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -12,17 +12,22 @@ class TlcIE(DiscoveryIE):
     IE_NAME = 'tlc.com'
     _VALID_URL = r'http://www\.tlc\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
 
-    _TEST = {
+    # DiscoveryIE has _TESTS
+    _TESTS = [{
         'url': 'http://www.tlc.com/tv-shows/cake-boss/videos/too-big-to-fly.htm',
-        'md5': 'c4038f4a9b44d0b5d74caaa64ed2a01a',
         'info_dict': {
-            'id': '853232',
+            'id': '104493',
             'ext': 'mp4',
-            'title': 'Cake Boss: Too Big to Fly',
+            'title': 'Too Big to Fly',
             'description': 'Buddy has taken on a high flying task.',
             'duration': 119,
+            'timestamp': 1393365060,
+            'upload_date': '20140225',
         },
-    }
+        'params': {
+            'skip_download': True,  # requires ffmpef
+        },
+    }]
 
 
 class TlcDeIE(InfoExtractor):

From edb99d4c18475ba27fae4f7d0ec6e3db9b574885 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 01:17:21 +0800
Subject: [PATCH 0342/2145] [instagram] Handling null values (fixes #5919)

I didn't add the test case here because it takes too much time. (7
minutes on my machine)
---
 youtube_dl/extractor/instagram.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b10755788..b92367a9d 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -100,7 +100,9 @@ class InstagramUserIE(InfoExtractor):
                 thumbnails_el = it.get('images', {})
                 thumbnail = thumbnails_el.get('thumbnail', {}).get('url')
 
-                title = it.get('caption', {}).get('text', it['id'])
+                # In some cases caption is null, which corresponds to None
+                # in python. As a result, it.get('caption', {}) gives None
+                title = (it.get('caption') or {}).get('text', it['id'])
 
                 entries.append({
                     'id': it['id'],

From e1ec93304dfcf385380feb95a3777c796cc49420 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 01:46:33 +0800
Subject: [PATCH 0343/2145] [instagram:user] Truncate title to 80 characters
 (#5919)

This is a workaround. Currently YoutubeDL.process_info() truncates
info_dict['title'] to 200 characters, but the implementation can't
handle wide characters.
---
 youtube_dl/extractor/instagram.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b92367a9d..3d78f78c4 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    limit_length,
+)
 
 
 class InstagramIE(InfoExtractor):
@@ -106,7 +109,7 @@ class InstagramUserIE(InfoExtractor):
 
                 entries.append({
                     'id': it['id'],
-                    'title': title,
+                    'title': limit_length(title, 80),
                     'formats': formats,
                     'thumbnail': thumbnail,
                     'webpage_url': it.get('link'),

From 788be3313df7ad020dc0a98bd5ed43a60120fb3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 13:32:04 +0800
Subject: [PATCH 0344/2145] [cnet] Fix theplatform vid extraction (fixes #5924)

---
 youtube_dl/extractor/cnet.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3145b3051..5dd69bff7 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CNETIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
@@ -25,7 +25,20 @@ class CNETIE(InfoExtractor):
         'params': {
             'skip_download': 'requires rtmpdump',
         }
-    }
+    }, {
+        'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
+        'info_dict': {
+            'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
+            'ext': 'flv',
+            'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
+            'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
+            'uploader': 'Ashley Esqueda',
+            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -42,7 +55,7 @@ class CNETIE(InfoExtractor):
             raise ExtractorError('Cannot find video data')
 
         mpx_account = data['config']['players']['default']['mpx_account']
-        vid = vdata['files']['rtmp']
+        vid = vdata['files'].get('rtmp', vdata['files']['hds'])
         tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
 
         video_id = vdata['id']

From 01e21b89eefc32bcc4a92c3a82658cee139b6b2c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 17:39:55 +0800
Subject: [PATCH 0345/2145] [noco] Skip invalid timestamps (closes #5826)

---
 youtube_dl/extractor/noco.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 664dc81d4..5bbd2dcf6 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -166,6 +166,10 @@ class NocoIE(InfoExtractor):
         self._sort_formats(formats)
 
         timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')
+
+        if timestamp is not None and timestamp < 0:
+            timestamp = None
+
         uploader = show.get('partner_name')
         uploader_id = show.get('partner_key')
         duration = float_or_none(show.get('duration_ms'), 1000)

From a55e36f48d1f0dc5454b144c7373361f284b9236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Jun 2015 21:05:17 +0600
Subject: [PATCH 0346/2145] [YoutubeDL] Handle out-of-range timestamps (#5826)

---
 youtube_dl/YoutubeDL.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aa6ec9d9a..b1f792d4e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1016,13 +1016,13 @@ class YoutubeDL(object):
             info_dict['display_id'] = info_dict['id']
 
         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
-            # Working around negative timestamps in Windows
-            # (see http://bugs.python.org/issue1646728)
-            if info_dict['timestamp'] < 0 and os.name == 'nt':
-                info_dict['timestamp'] = 0
-            upload_date = datetime.datetime.utcfromtimestamp(
-                info_dict['timestamp'])
-            info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+            # Working around out-of-range timestamp values (e.g. negative ones on Windows,
+            # see http://bugs.python.org/issue1646728)
+            try:
+                upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
+                info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+            except (ValueError, OverflowError, OSError):
+                pass
 
         if self.params.get('listsubtitles', False):
             if 'automatic_captions' in info_dict:

From 627b96482567c1525dddfcceae2c16ff53c18b6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jun 2015 11:41:17 +0800
Subject: [PATCH 0347/2145] [kickstarted] Extract thumbnails in embedded videos
 (#5929)

---
 youtube_dl/extractor/kickstarter.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 7d4b57056..1d391e69f 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -28,6 +28,14 @@ class KickStarterIE(InfoExtractor):
             'uploader': 'Pebble Technology',
             'title': 'Pebble iOS Notifications',
         }
+    }, {
+        'url': 'https://www.kickstarter.com/projects/1420158244/power-drive-2000/widget/video.html',
+        'info_dict': {
+            'id': '1420158244',
+            'ext': 'mp4',
+            'title': 'Power Drive 2000',
+        },
+        'expected_warnings': ['OpenGraph description'],
     }]
 
     def _real_extract(self, url):
@@ -48,10 +56,15 @@ class KickStarterIE(InfoExtractor):
                 'title': title,
             }
 
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
+        if thumbnail is None:
+            thumbnail = self._html_search_regex(
+                r'<img[^>]+class="[^"]+\s*poster\s*[^"]+"[^>]+src="([^"]+)"',
+                webpage, 'thumbnail image', fatal=False)
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
             'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': thumbnail,
         }

From e1b9322b091122b6f6832c70e3a845a84ee1764e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jun 2015 14:48:18 +0800
Subject: [PATCH 0348/2145] [youtube] Restricter DASH signature pattern

A problematic DASH url is:
https://manifest.googlevideo.com/api/manifest/dash/mm/35/key/yt5/ip/140.112.247.145/ms/pm/mv/s/mt/1433794435/id/o-AD2Od_dsOlAUYPu03ZsVWKSbGEbCJJrMp9vnXGhnyRhd/mn/sn-aigllm7r/sparams/as%2Chfr%2Cid%2Cip%2Cipbits%2Citag%2Cmm%2Cmn%2Cms%2Cmv%2Cnh%2Cpl%2Cplayback_host%2Crequiressl%2Csource%2Cexpire/fexp/9406009%2C9406821%2C9407575%2C9408142%2C9408420%2C9408710%2C9409121%2C9409208%2C9412514%2C9412780%2C9413208%2C9413426%2C9413476%2C9413503%2C9415304%2C9415753/upn/viDQrs8SnmE/as/fmp4_audio_clear%2Cwebm_audio_clear%2Cfmp4_sd_hd_clear%2Cwebm_sd_hd_clear%2Cwebm2_sd_hd_clear/playback_host/r4---sn-aigllm7r.googlevideo.com/ipbits/0/requiressl/yes/pl/20/itag/0/source/youtube/expire/1433824806/nh/EAQ/signature/81ABE6391E351BA495F5B041B00FF1257A353318.1A6E48ABB74E8F4AE73CA2CB1F963FC34E33DEE7/sver/3/hfr/1
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 419f7b019..083da777d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -785,7 +785,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
             return '/signature/%s' % dec_s
-        dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
+        dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
         dash_doc = self._download_xml(
             dash_manifest_url, video_id,
             note='Downloading DASH manifest',

From d9cf48e81e38f4bf151a8648c48d6e5233325b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 20:36:08 +0600
Subject: [PATCH 0349/2145] [spiegeltv] Extract all formats and prefer hls
 (Closes #5843)

---
 youtube_dl/extractor/spiegeltv.py | 45 ++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 359722ad6..08a5c4314 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -2,7 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import (
+    determine_ext,
+    float_or_none,
+)
 
 
 class SpiegeltvIE(InfoExtractor):
@@ -17,7 +21,7 @@ class SpiegeltvIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg$',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         }
     }, {
@@ -53,7 +57,35 @@ class SpiegeltvIE(InfoExtractor):
         server_json = self._download_json(
             'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
             video_id, note='Downloading server information')
-        server = server_json['streamingserver'][0]['endpoint']
+
+        format = '16x9' if is_wide else '4x3'
+
+        formats = []
+        for streamingserver in server_json['streamingserver']:
+            endpoint = streamingserver.get('endpoint')
+            if not endpoint:
+                continue
+            play_path = 'mp4:%s_spiegeltv_0500_%s.m4v' % (uuid, format)
+            if endpoint.startswith('rtmp'):
+                formats.append({
+                    'url': endpoint,
+                    'format_id': 'rtmp',
+                    'app': compat_urllib_parse_urlparse(endpoint).path[1:],
+                    'play_path': play_path,
+                    'player_path': 'http://prod-static.spiegel.tv/frontend-076.swf',
+                    'ext': 'flv',
+                    'rtmp_live': True,
+                })
+            elif determine_ext(endpoint) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    endpoint.replace('[video]', play_path),
+                    video_id, 'm4v',
+                    preference=1,  # Prefer hls since it allows to workaround georestriction
+                    m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': endpoint,
+                })
 
         thumbnails = []
         for image in media_json['images']:
@@ -65,17 +97,12 @@ class SpiegeltvIE(InfoExtractor):
 
         description = media_json['subtitle']
         duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
-        format = '16x9' if is_wide else '4x3'
-
-        url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
 
         return {
             'id': video_id,
             'title': title,
-            'url': url,
-            'ext': 'm4v',
             'description': description,
             'duration': duration,
             'thumbnails': thumbnails,
-            'rtmp_live': True,
+            'formats': formats,
         }

From 9bf99891d08d166ac1d81b652dc487bb940fa685 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 21:23:53 +0600
Subject: [PATCH 0350/2145] [cbs] Add support for colbertlateshow (Closes
 #5888)

---
 youtube_dl/extractor/cbs.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 1ceb9d8d9..89614a3c9 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class CBSIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)/(?P<id>[^/]+)/.*'
+    _VALID_URL = r'https?://(?:(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/(?P<id>[^/]+)/.*'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
@@ -34,12 +34,18 @@ class CBSIE(InfoExtractor):
             'skip_download': True,
         },
         '_skip': 'Blocked outside the US',
+    }, {
+        'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
+        'only_matching': True,
+    }, {
+        'url': 'http://colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         real_id = self._search_regex(
-            r"video\.settings\.pid\s*=\s*'([^']+)';",
+            [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
         return self.url_result('theplatform:%s' % real_id)

From 9d581f3d5224140ca35ebd06d614b929e9252cd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 21:39:45 +0600
Subject: [PATCH 0351/2145] [cbs] Extract display_id

---
 youtube_dl/extractor/cbs.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 89614a3c9..75fffb156 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,12 +4,13 @@ from .common import InfoExtractor
 
 
 class CBSIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/(?P<id>[^/]+)/.*'
+    _VALID_URL = r'https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
         'info_dict': {
             'id': '4JUVEwq3wUT7',
+            'display_id': 'connect-chat-feat-garth-brooks',
             'ext': 'flv',
             'title': 'Connect Chat feat. Garth Brooks',
             'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
@@ -24,6 +25,7 @@ class CBSIE(InfoExtractor):
         'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/',
         'info_dict': {
             'id': 'WWF_5KqY3PK1',
+            'display_id': 'st-vincent',
             'ext': 'flv',
             'title': 'Live on Letterman - St. Vincent',
             'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.',
@@ -38,14 +40,19 @@ class CBSIE(InfoExtractor):
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
         'only_matching': True,
     }, {
-        'url': 'http://colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
+        'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
         real_id = self._search_regex(
             [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
-        return self.url_result('theplatform:%s' % real_id)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': 'theplatform:%s' % real_id,
+            'display_id': display_id,
+        }

From 6e054aacca2ac44413ed37ee8b1d63a09c8b4ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 23:07:22 +0600
Subject: [PATCH 0352/2145] [theplatform] Take care of /select/media URLs
 (Closes #5746)

---
 youtube_dl/extractor/theplatform.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 92731ad3d..48c6ff03f 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -26,7 +26,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
+           (?:(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)|(?P<media>(?:[^/]+/)+select/media/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{
@@ -56,6 +56,17 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
+        'info_dict': {
+            'id': 'yMBg9E8KFxZD',
+            'ext': 'mp4',
+            'description': 'md5:644ad9188d655b742f942bf2e06b002d',
+            'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
+        }
+    }, {
+        'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -85,6 +96,11 @@ class ThePlatformIE(InfoExtractor):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
+        path = provider_id
+        if mobj.group('media'):
+            path += '/media'
+        path += '/' + video_id
+
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
         elif mobj.group('config'):
@@ -94,8 +110,7 @@ class ThePlatformIE(InfoExtractor):
             config = self._download_json(config_url, video_id, 'Downloading config')
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = ('http://link.theplatform.com/s/{0}/{1}/meta.smil?'
-                        'format=smil&mbr=true'.format(provider_id, video_id))
+            smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
 
         sig = smuggled_data.get('sig')
         if sig:
@@ -112,7 +127,7 @@ class ThePlatformIE(InfoExtractor):
         else:
             raise ExtractorError(error_msg, expected=True)
 
-        info_url = 'http://link.theplatform.com/s/{0}/{1}?format=preview'.format(provider_id, video_id)
+        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 

From bd5bc0cd5af257abf7a1a4c14a9dd39c4f97e622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 23:12:13 +0600
Subject: [PATCH 0353/2145] [theplatform] Check for /select/media URLs first
 (#5746)

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 48c6ff03f..83d833e30 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -26,7 +26,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)|(?P<media>(?:[^/]+/)+select/media/))?
+           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 70219b0f4371fe54cc72d025ce06fc4691ba12fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 9 Jun 2015 23:49:11 +0200
Subject: [PATCH 0354/2145] [youtube:playlist] Use an iterator for the entries
 (closes #5935)

So that '--playlist-end' downloads only the required pages.
---
 youtube_dl/extractor/youtube.py | 47 ++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 083da777d..3448bec4f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1290,7 +1290,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     def _extract_playlist(self, playlist_id):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
-        more_widget_html = content_html = page
 
         for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
             match = match.strip()
@@ -1310,36 +1309,36 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 self.report_warning('Youtube gives an alert message: ' + match)
 
         # Extract the video ids from the playlist pages
-        ids = []
+        def _entries():
+            more_widget_html = content_html = page
+            for page_num in itertools.count(1):
+                matches = re.finditer(self._VIDEO_RE, content_html)
+                # We remove the duplicates and the link with index 0
+                # (it's not the first video of the playlist)
+                new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
+                for vid_id in new_ids:
+                    yield self.url_result(vid_id, 'Youtube', video_id=vid_id)
 
-        for page_num in itertools.count(1):
-            matches = re.finditer(self._VIDEO_RE, content_html)
-            # We remove the duplicates and the link with index 0
-            # (it's not the first video of the playlist)
-            new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
-            ids.extend(new_ids)
+                mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+                if not mobj:
+                    break
 
-            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-            if not mobj:
-                break
-
-            more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
-                'Downloading page #%s' % page_num,
-                transform_source=uppercase_escape)
-            content_html = more['content_html']
-            if not content_html.strip():
-                # Some webpages show a "Load more" button but they don't
-                # have more videos
-                break
-            more_widget_html = more['load_more_widget_html']
+                more = self._download_json(
+                    'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+                    'Downloading page #%s' % page_num,
+                    transform_source=uppercase_escape)
+                content_html = more['content_html']
+                if not content_html.strip():
+                    # Some webpages show a "Load more" button but they don't
+                    # have more videos
+                    break
+                more_widget_html = more['load_more_widget_html']
 
         playlist_title = self._html_search_regex(
             r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
             page, 'title')
 
-        url_results = self._ids_to_results(ids)
-        return self.playlist_result(url_results, playlist_id, playlist_title)
+        return self.playlist_result(_entries(), playlist_id, playlist_title)
 
     def _real_extract(self, url):
         # Extract playlist id

From d84f1d14b526c4a5359117a58f25691a3da4c97e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 9 Jun 2015 22:08:16 -0300
Subject: [PATCH 0355/2145] Adds support for XviD output with extra
 parametrization

As the "LG Time Machine" (a (not so) smart TV) has a limitation for video dimensions (as for codecs), I take to implement an extra parameter `--pp-params` where we can send extra parameterization for the video converter (post-processor).

Example:
```
$ youtube-dl --recode-video=xvid --pp-params='-s 720x480' -c https://www.youtube.com/watch?v=BE7Qoe2ZiXE
```
That works fine on a 4yo LG Time Machine.

Closes #5733
---
 README.md                          |  3 ++-
 youtube_dl/__init__.py             |  5 ++++-
 youtube_dl/options.py              |  6 +++++-
 youtube_dl/postprocessor/ffmpeg.py | 14 ++++++++++----
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index f3d83c89f..726ec9cf2 100644
--- a/README.md
+++ b/README.md
@@ -213,7 +213,8 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
+    --pp-params                      Extra parameters for video post-processor. The params will be splited on spaces.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ace17857c..5b28e4817 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -169,8 +169,10 @@ def _real_main(argv=None):
         if not opts.audioquality.isdigit():
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
-        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
+    if opts.pp_params is not None:
+        opts.pp_params = opts.pp_params.split()
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
@@ -227,6 +229,7 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
+            'extra_params': opts.pp_params
         })
     if opts.convertsubtitles:
         postprocessors.append({
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..ceb4b5f38 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -686,7 +686,11 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
-        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
+    postproc.add_option(
+        '--pp-params',
+        dest='pp_params', default=None,
+        help='Extra parameters for video post-processor. The params will be splited on spaces.')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index cc65b34e7..a696b12b4 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -287,22 +287,28 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None):
+    def __init__(self, downloader=None, preferedformat=None, extra_params=[]):
         super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
+        self._extra_params = extra_params
 
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
-        outpath = prefix + sep + self._preferedformat
+        ext = self._preferedformat
+        options = self._extra_params
+        if self._preferedformat == 'xvid':
+            ext = 'avi'
+            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
+        outpath = prefix + sep + ext
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
-        self.run_ffmpeg(path, outpath, [])
+        self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath
         information['format'] = self._preferedformat
-        information['ext'] = self._preferedformat
+        information['ext'] = ext
         return [path], information
 
 

From 0c8662d2b6f033ad42f1cc97989d4975629b524b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 13:40:41 +0800
Subject: [PATCH 0356/2145] [youtube] Fix a TypeError caused by
 4da31bd56629054497634d041035e4bd6fcfacbb

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2424ac2c0..a1906eef6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -816,7 +816,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
-                    if len(segment_list):
+                    if segment_list is not None:
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],

From 93dfcb9357b400b4d7e353d0a9db0e0194135b19 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 13:44:54 +0800
Subject: [PATCH 0357/2145] [downloader/dash] Do not pollute ```self```

---
 youtube_dl/downloader/dash.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 5f14658ba..cd84e0b07 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -16,14 +16,14 @@ class DashSegmentsFD(FileDownloader):
         base_url = info_dict['url']
         segment_urls = info_dict['segment_urls']
 
-        self.byte_counter = 0
+        byte_counter = 0
 
         def append_url_to_file(outf, target_url, target_name):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
             req = compat_urllib_request.Request(target_url)
             data = self.ydl.urlopen(req).read()
             outf.write(data)
-            self.byte_counter += len(data)
+            return len(data)
 
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
@@ -35,15 +35,16 @@ class DashSegmentsFD(FileDownloader):
                 outf, combine_url(base_url, info_dict['initialization_url']),
                 'initialization segment')
             for i, segment_url in enumerate(segment_urls):
-                append_url_to_file(
+                segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),
                     'segment %d / %d' % (i + 1, len(segment_urls)))
+                byte_counter += segment_len
 
         self.try_rename(tmpfilename, filename)
 
         self._hook_progress({
-            'downloaded_bytes': self.byte_counter,
-            'total_bytes': self.byte_counter,
+            'downloaded_bytes': byte_counter,
+            'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
         })

From 7ebd5376feb493edd0bc04abd07bba89397b7307 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:15:20 +0800
Subject: [PATCH 0358/2145] [nfl] Relax _VALID_URL (fixes #5940)

---
 youtube_dl/extractor/nfl.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 2684dd250..dc54634a5 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -19,7 +19,7 @@ class NFLIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
         (?:.+?/)*
-        (?P<id>(?:\d[a-z]{2}\d{13}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
+        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
     _TESTS = [
         {
             'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
@@ -58,6 +58,10 @@ class NFLIE(InfoExtractor):
                 'upload_date': '20150202',
             },
         },
+        {
+            'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
+            'only_matching': True,
+        }
     ]
 
     @staticmethod

From 5bf3276e8d6ee7d017c8be04414398752cd9cdf3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:45:54 +0800
Subject: [PATCH 0359/2145] [downloader/dash] Add testing facility

---
 youtube_dl/downloader/dash.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index cd84e0b07..a4685d307 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -16,12 +16,21 @@ class DashSegmentsFD(FileDownloader):
         base_url = info_dict['url']
         segment_urls = info_dict['segment_urls']
 
+        is_test = self.params.get('test', False)
+        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
         byte_counter = 0
 
-        def append_url_to_file(outf, target_url, target_name):
+        def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
             req = compat_urllib_request.Request(target_url)
+            if remaining_bytes is not None:
+                req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
+
             data = self.ydl.urlopen(req).read()
+
+            if remaining_bytes is not None:
+                data = data[:remaining_bytes]
+
             outf.write(data)
             return len(data)
 
@@ -37,8 +46,13 @@ class DashSegmentsFD(FileDownloader):
             for i, segment_url in enumerate(segment_urls):
                 segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),
-                    'segment %d / %d' % (i + 1, len(segment_urls)))
+                    'segment %d / %d' % (i + 1, len(segment_urls)),
+                    remaining_bytes)
                 byte_counter += segment_len
+                if remaining_bytes is not None:
+                    remaining_bytes -= segment_len
+                    if remaining_bytes <= 0:
+                        break
 
         self.try_rename(tmpfilename, filename)
 

From 8a1a26ce4c64d7a2c142718fc56f46d9a1c2c4f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:47:02 +0800
Subject: [PATCH 0360/2145] [youtube] Add a test for the DASH segment
 downloader

---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a1906eef6..939f5e61f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -516,6 +516,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # DASH manifest with segment_list
+        {
+            'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
+            'md5': '8ce563a1d667b599d21064e982ab9e31',
+            'info_dict': {
+                'id': 'CsmdDsKjzN8',
+                'ext': 'mp4',
+                'upload_date': '20150510',
+                'uploader': 'Airtek',
+                'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
+                'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
+                'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
+            },
+            'params': {
+                'youtube_include_dash_manifest': True,
+                'format': '135',  # bestvideo
+            }
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From eb8be1fe76a9fbc285e6c957b3fdd5c05135ae3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 10 Jun 2015 14:12:43 +0200
Subject: [PATCH 0361/2145] [rtbf] Extract all formats (closes #5947)

---
 youtube_dl/extractor/rtbf.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index 5a381d9ce..e4215d546 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -21,6 +21,13 @@ class RTBFIE(InfoExtractor):
         }
     }
 
+    _QUALITIES = [
+        ('mobile', 'mobile'),
+        ('web', 'SD'),
+        ('url', 'MD'),
+        ('high', 'HD'),
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -32,14 +39,21 @@ class RTBFIE(InfoExtractor):
                 r'data-video="([^"]+)"', webpage, 'data video')),
             video_id)
 
-        video_url = data.get('downloadUrl') or data.get('url')
-
         if data.get('provider').lower() == 'youtube':
+            video_url = data.get('downloadUrl') or data.get('url')
             return self.url_result(video_url, 'Youtube')
+        formats = []
+        for key, format_id in self._QUALITIES:
+            format_url = data['sources'].get(key)
+            if format_url:
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                })
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
             'thumbnail': data.get('thumbnail'),

From f98470df690d053e45691ede2751ab6a4063082b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Jun 2015 23:01:12 +0600
Subject: [PATCH 0362/2145] [bilibili] Fix FutureWarning

---
 youtube_dl/extractor/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 2103ed73a..bf60450c2 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -105,7 +105,7 @@ class BiliBiliIE(InfoExtractor):
                 'filesize': int_or_none(
                     lq_durl.find('./size'), get_attr='text'),
             }]
-            if hq_durl:
+            if hq_durl is not None:
                 formats.append({
                     'format_id': 'hq',
                     'quality': 2,

From a9d56c684319eaf8b9494bd8d2dc9d0f40485254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 19:03:22 +0600
Subject: [PATCH 0363/2145] [rtlnl] Improve _VALID_URL (#5950)

---
 youtube_dl/extractor/rtlnl.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index cfce4550a..41d202c28 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -12,10 +12,10 @@ class RtlNlIE(InfoExtractor):
     IE_NAME = 'rtl.nl'
     IE_DESC = 'rtl.nl and rtlxl.nl'
     _VALID_URL = r'''(?x)
-        https?://(www\.)?
+        https?://(?:www\.)?
         (?:
             rtlxl\.nl/\#!/[^/]+/|
-            rtl\.nl/system/videoplayer/[^?#]+?/video_embed\.html\#uuid=
+            rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -43,6 +43,9 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 97b570a94cc2387153af525f781e144bb4bb791e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 19:04:12 +0600
Subject: [PATCH 0364/2145] [generic] Improve rtl.nl embeds detection (Closes
 #5950)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 759691365..75526384f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,7 +1073,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded rtl.nl player
         matches = re.findall(
-            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
+            r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
             webpage)
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')

From ff0f0b9172e432ebbfca88da91278554eb47c307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 22:18:08 +0600
Subject: [PATCH 0365/2145] [tube8] Fix extraction (Closes #5952)

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 6ca8840b0..c9cb69333 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -47,7 +47,7 @@ class Tube8IE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
-            r'flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
+            r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'))
 
         video_url = flashvars['video_url']
         if flashvars.get('encrypted') is True:

From 99ac0390f559aa6dd09ffd8a15b9b562fda5f363 Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Mon, 8 Jun 2015 05:58:41 +0300
Subject: [PATCH 0366/2145] [fivetv] Add extractor (Closes #5794)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fivetv.py   | 67 ++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/fivetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 67eb96057..d10275d03 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -152,6 +152,7 @@ from .fc2 import FC2IE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
+from .fivetv import FiveTVIE
 from .fktv import (
     FKTVIE,
     FKTVPosteckeIE,
diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
new file mode 100644
index 000000000..e47383b39
--- /dev/null
+++ b/youtube_dl/extractor/fivetv.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class FiveTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?5-tv\.ru/[^/]*/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://5-tv.ru/news/96814/',
+            'md5': 'bbff554ad415ecf5416a2f48c22d9283',
+            'info_dict': {
+                'id': '96814',
+                'ext': 'mp4',
+                'title': 'Россияне выбрали имя для общенациональной платежной системы',
+                'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'width': 480,
+                'height': 360,
+                'duration': 180,
+            },
+        },
+        {
+            'url': 'http://5-tv.ru/video/1021729/',
+            'md5': '299c8b72960efc9990acd2c784dc2296',
+            'info_dict': {
+                'id': '1021729',
+                'ext': 'mp4',
+                'title': '3D принтер',
+                'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'width': 480,
+                'height': 360,
+                'duration': 180,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_link = self._search_regex(
+            r'(<a.*?class="videoplayer">)', webpage, 'video link')
+
+        url = self._search_regex(r'href="([^"]+)"', video_link, 'video url')
+        width = int_or_none(self._search_regex(
+            r'width:(\d+)px', video_link, 'width', default=None, fatal=False))
+        height = int_or_none(self._search_regex(
+            r'height:(\d+)px', video_link, 'height', default=None, fatal=False))
+        duration = int_or_none(self._og_search_property(
+            'video:duration', webpage, 'duration'))
+        return {
+            'id': video_id,
+            'url': url,
+            'width': width,
+            'height': height,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+        }

From 87446dc6186c7e4247fd7f9bc1046ef41f5d1a0f Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sun, 7 Jun 2015 17:25:30 +0300
Subject: [PATCH 0367/2145] [tvc] Add extractor (Closes #5795)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tvc.py      | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tvc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 67eb96057..8c4e12904 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -582,6 +582,7 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv4 import TV4IE
+from .tvc import TVCIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
new file mode 100644
index 000000000..b62ab857c
--- /dev/null
+++ b/youtube_dl/extractor/tvc.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    str_or_none,
+)
+
+
+class TVCIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
+            'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba',
+            'info_dict': {
+                'id': '74622',
+                'display_id': '39702',
+                'ext': 'mp4',
+                'title': 'События. "События". Эфир от 22.05.2015 14:30',
+                'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 1122,
+            },
+        },
+        {
+            'url': 'http://www.tvc.ru/news/show/id/69944',
+            'md5': 'b173128ee7b88b5b06c84e5f7880909f',
+            'info_dict': {
+                'id': '75399',
+                'display_id': '69944',
+                'ext': 'mp4',
+                'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
+                'description': 'md5:f675c8eaf23aab9df542d31773ed6518',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 278,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._og_search_video_url(webpage)
+
+        video_id = self._search_regex(
+            r'video/iframe/id/(\d+)/', video_url, 'video id')
+
+        video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id)
+
+        video_json = self._download_json(video_json_url, video_id)
+
+        formats = []
+        for info in video_json.get('path', {}).get('quality', []):
+            format_id = self._search_regex(
+                r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id',
+                fatal=False)
+            formats.append({
+                'format_id': str_or_none(format_id),
+                'url': info.get('url'),
+                'width': int_or_none(info.get('width')),
+                'height': int_or_none(info.get('height')),
+                'tbr': int_or_none(info.get('bitrate')),
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int_or_none(video_json.get('duration')),
+            'formats': formats,
+        }

From 9f15bdabc85add582d78a6dd57cfbb56cb33baff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:13:36 +0600
Subject: [PATCH 0368/2145] [tvc] Separate embed extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/tvc.py      | 125 ++++++++++++++++++-------------
 2 files changed, 77 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6dc3cbff4..a8d3a8928 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -584,7 +584,10 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv4 import TV4IE
-from .tvc import TVCIE
+from .tvc import (
+    TVCIE,
+    TVCEmbedIE,
+)
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index b62ab857c..0055f9598 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -3,77 +3,98 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     int_or_none,
-    str_or_none,
 )
 
 
-class TVCIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
-            'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba',
-            'info_dict': {
-                'id': '74622',
-                'display_id': '39702',
-                'ext': 'mp4',
-                'title': 'События. "События". Эфир от 22.05.2015 14:30',
-                'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'duration': 1122,
-            },
+class TVCEmbedIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
+        'md5': 'bbc5ff531d1e90e856f60fc4b3afd708',
+        'info_dict': {
+            'id': '74622',
+            'ext': 'mp4',
+            'title': 'События. "События". Эфир от 22.05.2015 14:30',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1122,
         },
-        {
-            'url': 'http://www.tvc.ru/news/show/id/69944',
-            'md5': 'b173128ee7b88b5b06c84e5f7880909f',
-            'info_dict': {
-                'id': '75399',
-                'display_id': '69944',
-                'ext': 'mp4',
-                'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
-                'description': 'md5:f675c8eaf23aab9df542d31773ed6518',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'duration': 278,
-            },
-        },
-    ]
+    }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_url = self._og_search_video_url(webpage)
-
-        video_id = self._search_regex(
-            r'video/iframe/id/(\d+)/', video_url, 'video id')
-
-        video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id)
-
-        video_json = self._download_json(video_json_url, video_id)
+        video = self._download_json(
+            'http://www.tvc.ru/video/json/id/%s' % video_id, video_id)
 
         formats = []
-        for info in video_json.get('path', {}).get('quality', []):
+        for info in video.get('path', {}).get('quality', []):
+            video_url = info.get('url')
+            if not video_url:
+                continue
             format_id = self._search_regex(
-                r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id',
-                fatal=False)
+                r'cdnvideo/([^/]+?)(?:-[^/]+?)?/', video_url,
+                'format id', default=None)
             formats.append({
-                'format_id': str_or_none(format_id),
-                'url': info.get('url'),
+                'url': video_url,
+                'format_id': format_id,
                 'width': int_or_none(info.get('width')),
                 'height': int_or_none(info.get('height')),
                 'tbr': int_or_none(info.get('bitrate')),
             })
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'display_id': display_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': int_or_none(video_json.get('duration')),
+            'title': video['title'],
+            'thumbnail': video.get('picture'),
+            'duration': int_or_none(video.get('duration')),
             'formats': formats,
         }
+
+
+class TVCIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
+    _TESTS = [{
+        'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
+        'info_dict': {
+            'id': '74622',
+            'ext': 'mp4',
+            'title': 'События. "События". Эфир от 22.05.2015 14:30',
+            'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1122,
+        },
+    }, {
+        'url': 'http://www.tvc.ru/news/show/id/69944',
+        'info_dict': {
+            'id': '75399',
+            'ext': 'mp4',
+            'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
+            'description': 'md5:f2098f71e21f309e89f69b525fd9846e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 278,
+        },
+    }, {
+        'url': 'http://www.tvc.ru/channel/brand/id/47/show/episodes#',
+        'info_dict': {
+            'id': '2185',
+            'ext': 'mp4',
+            'title': 'Ещё не поздно. Эфир от 03.08.2013',
+            'description': 'md5:51fae9f3f8cfe67abce014e428e5b027',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 3316,
+        },
+    }]
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, self._match_id(url))
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'TVCEmbed',
+            'url': self._og_search_video_url(webpage),
+            'title': clean_html(self._og_search_title(webpage)),
+            'description': clean_html(self._og_search_description(webpage)),
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }

From 29902c8ec016a7128557d47a7413e82d4e022f01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:22:23 +0600
Subject: [PATCH 0369/2145] [tvc:embed] Add embed extraction routine

---
 youtube_dl/extractor/tvc.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 0055f9598..756fec732 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
@@ -22,6 +24,13 @@ class TVCEmbedIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 494f20cbdca8e76e3cb452bb0feabcb855d9b4a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:22:46 +0600
Subject: [PATCH 0370/2145] [extractor/generic] Add support for tvc embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 75526384f..c797c4b52 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -34,6 +34,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
+from .tvc import TVCEmbedIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
@@ -1301,6 +1302,11 @@ class GenericIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
+        # Look for embedded TVC player
+        rutv_url = TVCEmbedIE._extract_url(webpage)
+        if rutv_url:
+            return self.url_result(rutv_url, 'TVCEmbed')
+
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
         if sportbox_urls:

From 954c1d05299ae7c6a51db46c1ac33ddf150266c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:24:13 +0600
Subject: [PATCH 0371/2145] [tvc] Refactor extractor names

---
 youtube_dl/extractor/__init__.py | 2 +-
 youtube_dl/extractor/generic.py  | 4 ++--
 youtube_dl/extractor/tvc.py      | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a8d3a8928..18b1c5e54 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -586,7 +586,7 @@ from .tv2 import (
 from .tv4 import TV4IE
 from .tvc import (
     TVCIE,
-    TVCEmbedIE,
+    TVCArticleIE,
 )
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c797c4b52..507e4a571 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -34,7 +34,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
-from .tvc import TVCEmbedIE
+from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
@@ -1303,7 +1303,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(rutv_url, 'RUTV')
 
         # Look for embedded TVC player
-        rutv_url = TVCEmbedIE._extract_url(webpage)
+        rutv_url = TVCIE._extract_url(webpage)
         if rutv_url:
             return self.url_result(rutv_url, 'TVCEmbed')
 
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 756fec732..36c2a3196 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class TVCEmbedIE(InfoExtractor):
+class TVCIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
@@ -63,7 +63,7 @@ class TVCEmbedIE(InfoExtractor):
         }
 
 
-class TVCIE(InfoExtractor):
+class TVCArticleIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
     _TESTS = [{
         'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',

From 5ccddb7ecfb1015038f2616dd7e0da78a4365c89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:25:26 +0600
Subject: [PATCH 0372/2145] [tvc] Fix ie_key

---
 youtube_dl/extractor/tvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 36c2a3196..6b5d80aee 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -101,7 +101,7 @@ class TVCArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         return {
             '_type': 'url_transparent',
-            'ie_key': 'TVCEmbed',
+            'ie_key': 'TVC',
             'url': self._og_search_video_url(webpage),
             'title': clean_html(self._og_search_title(webpage)),
             'description': clean_html(self._og_search_description(webpage)),

From 2da09ff8b0de3c27a16d9096f5d28d03f44fcf70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:26:31 +0600
Subject: [PATCH 0373/2145] [extractor/generic] Fix tvc ie_key

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 507e4a571..66aceefb8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor):
         # Look for embedded TVC player
         rutv_url = TVCIE._extract_url(webpage)
         if rutv_url:
-            return self.url_result(rutv_url, 'TVCEmbed')
+            return self.url_result(rutv_url, 'TVC')
 
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)

From f37bdbe537134f1ece0819e2aa677b1fec0c1cc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:28:45 +0600
Subject: [PATCH 0374/2145] [extractor/generic] Add test for tvc embed

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 66aceefb8..6be9e6329 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -292,6 +292,15 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # TVC embed
+        {
+            'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
+            'info_dict': {
+                'id': '55304',
+                'ext': 'mp4',
+                'title': 'Дошкольное воспитание',
+            },
+        },
         # SportBox embed
         {
             'url': 'http://www.vestifinance.ru/articles/25753',

From 499a077761b1577857952dc3b541c9f61a8bcade Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 17:48:42 +0600
Subject: [PATCH 0375/2145] [5tv] Improve

---
 youtube_dl/extractor/fivetv.py | 115 +++++++++++++++++++--------------
 1 file changed, 68 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
index e47383b39..13fbc4da2 100644
--- a/youtube_dl/extractor/fivetv.py
+++ b/youtube_dl/extractor/fivetv.py
@@ -1,67 +1,88 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
+from ..utils import int_or_none
 
 
 class FiveTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?5-tv\.ru/[^/]*/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://5-tv.ru/news/96814/',
-            'md5': 'bbff554ad415ecf5416a2f48c22d9283',
-            'info_dict': {
-                'id': '96814',
-                'ext': 'mp4',
-                'title': 'Россияне выбрали имя для общенациональной платежной системы',
-                'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 360,
-                'duration': 180,
-            },
+    _VALID_URL = r'''(?x)
+                    http://
+                        (?:www\.)?5-tv\.ru/
+                        (?:
+                            (?:[^/]+/)+(?P<id>\d+)|
+                            (?P<path>[^/?#]+)(?:[/?#])?
+                        )
+                    '''
+
+    _TESTS = [{
+        'url': 'http://5-tv.ru/news/96814/',
+        'md5': 'bbff554ad415ecf5416a2f48c22d9283',
+        'info_dict': {
+            'id': '96814',
+            'ext': 'mp4',
+            'title': 'Россияне выбрали имя для общенациональной платежной системы',
+            'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 180,
         },
-        {
-            'url': 'http://5-tv.ru/video/1021729/',
-            'md5': '299c8b72960efc9990acd2c784dc2296',
-            'info_dict': {
-                'id': '1021729',
-                'ext': 'mp4',
-                'title': '3D принтер',
-                'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 360,
-                'duration': 180,
-            },
+    }, {
+        'url': 'http://5-tv.ru/video/1021729/',
+        'info_dict': {
+            'id': '1021729',
+            'ext': 'mp4',
+            'title': '3D принтер',
+            'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 180,
         },
-    ]
+    }, {
+        'url': 'http://www.5-tv.ru/glavnoe/#itemDetails',
+        'info_dict': {
+            'id': 'glavnoe',
+            'ext': 'mp4',
+            'title': 'Итоги недели с 8 по 14 июня 2015 года',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }, {
+        'url': 'http://www.5-tv.ru/glavnoe/broadcasts/508645/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/films/1507502/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/programs/broadcast/508713/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/angel/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.5-tv.ru/schedule/?iframe=true&width=900&height=450',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('path')
 
         webpage = self._download_webpage(url, video_id)
 
-        video_link = self._search_regex(
-            r'(<a.*?class="videoplayer">)', webpage, 'video link')
+        video_url = self._search_regex(
+            r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"',
+            webpage, 'video url')
 
-        url = self._search_regex(r'href="([^"]+)"', video_link, 'video url')
-        width = int_or_none(self._search_regex(
-            r'width:(\d+)px', video_link, 'width', default=None, fatal=False))
-        height = int_or_none(self._search_regex(
-            r'height:(\d+)px', video_link, 'height', default=None, fatal=False))
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
         duration = int_or_none(self._og_search_property(
-            'video:duration', webpage, 'duration'))
+            'video:duration', webpage, 'duration', default=None))
+
         return {
             'id': video_id,
-            'url': url,
-            'width': width,
-            'height': height,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'url': video_url,
+            'title': title,
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
         }

From b859971873915df55668a59a18ccfd259c20800e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 18:15:30 +0600
Subject: [PATCH 0376/2145] [extractor/generic] Rename tvc embed url variable

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6be9e6329..357d58cea 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1312,9 +1312,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(rutv_url, 'RUTV')
 
         # Look for embedded TVC player
-        rutv_url = TVCIE._extract_url(webpage)
-        if rutv_url:
-            return self.url_result(rutv_url, 'TVC')
+        tvc_url = TVCIE._extract_url(webpage)
+        if tvc_url:
+            return self.url_result(tvc_url, 'TVC')
 
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)

From 9872d3110c0d3027dac856e005299f3682ef23ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 18:37:09 +0600
Subject: [PATCH 0377/2145] [extractor/generic] Add support for tvigle embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 357d58cea..40d869c53 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1321,6 +1321,12 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        # Look for embedded Tvigle player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Tvigle')
+
         # Look for embedded TED player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)

From d22dec74ffa2a53a1c04770af37d39f384f3d56c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 19:20:12 +0600
Subject: [PATCH 0378/2145] Add `--force-generic-extractor`

For some extractors that are hard to workout a good _VALID_URL we use very vague and unrestrictive ones,
e.g. just allowing anything after hostname and capturing part of URL as id.
If some of these extractors happen to have an video embed of some different hoster or platform
and this scenario was not handled in extractor itself we end up with inability to download this embed
until extractor is fixed to support embed of this kind.
Forcing downloader to use the generic extractor can be a neat temporary solution for this problem.
Example: FiveTV extractor with Tvigle embed - http://www.5-tv.ru/rabota/broadcasts/48/
---
 youtube_dl/YoutubeDL.py         | 6 ++++++
 youtube_dl/__init__.py          | 1 +
 youtube_dl/extractor/generic.py | 4 +++-
 youtube_dl/options.py           | 4 ++++
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..4b801a917 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -139,6 +139,7 @@ class YoutubeDL(object):
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
+    force_generic_extractor: Force downloader to use the generic extractor
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
@@ -282,6 +283,7 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
+        self._force_generic_extractor_required = params.get('force_generic_extractor', False)
         self.params = params
         self.cache = Cache(self)
 
@@ -633,6 +635,10 @@ class YoutubeDL(object):
         extra_info is a dict containing the extra values to add to each result
         '''
 
+        if not ie_key and self._force_generic_extractor_required:
+            self._force_generic_extractor_required = False
+            ie_key = 'Generic'
+
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]
         else:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ace17857c..215b616de 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -293,6 +293,7 @@ def _real_main(argv=None):
         'autonumber_size': opts.autonumber_size,
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
+        'force_generic_extractor': opts.force_generic_extractor,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
         'retries': opts_retries,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40d869c53..3d672197c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -977,7 +977,9 @@ class GenericIE(InfoExtractor):
                 'upload_date': upload_date,
             }
 
-        if not self._downloader.params.get('test', False) and not is_intentional:
+        if (not self._downloader.params.get('test', False) and
+                not is_intentional and
+                not self._downloader.params.get('force_generic_extractor', False)):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if not full_response:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..096ab6137 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -150,6 +150,10 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
+    general.add_option(
+        '--force-generic-extractor',
+        action='store_true', dest='force_generic_extractor', default=False,
+        help='Force extraction to use the generic extractor')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',

From 3d535e047162af021b3df6086f9a90d0cb0b6100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 19:31:52 +0600
Subject: [PATCH 0379/2145] [tvc] Fix embed regex

---
 youtube_dl/extractor/tvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 6b5d80aee..3a4f393fc 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -27,7 +27,7 @@ class TVCIE(InfoExtractor):
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http:)?//(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 185dbc49749ca81fbb0a61a78c6dd35f2c32b15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:13:14 +0600
Subject: [PATCH 0380/2145] [prosiebensat1] Fix rtmp extraction (Closes #5962)

---
 youtube_dl/extractor/prosiebensat1.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 255d4abc1..6b13eb605 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -206,8 +206,8 @@ class ProSiebenSat1IE(InfoExtractor):
     def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
-        access_token = 'testclient'
-        client_name = 'kolibri-1.2.5'
+        access_token = 'prosieben'
+        client_name = 'kolibri-1.12.6'
         client_location = url
 
         videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
@@ -275,13 +275,17 @@ class ProSiebenSat1IE(InfoExtractor):
         for source in urls_sources:
             protocol = source['protocol']
             if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', source['url'])
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source['url'])
                 if not mobj:
                     continue
+                path = mobj.group('path')
+                mp4colon_index = path.rfind('mp4:')
+                app = path[:mp4colon_index]
+                play_path = path[mp4colon_index:]
                 formats.append({
-                    'url': mobj.group('url'),
-                    'app': mobj.group('app'),
-                    'play_path': mobj.group('playpath'),
+                    'url': '%s/%s' % (mobj.group('url'), app),
+                    'app': app,
+                    'play_path': play_path,
                     'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
                     'page_url': 'http://www.prosieben.de',
                     'vbr': fix_bitrate(source['bitrate']),

From 8b6c896c4b60fe13b30227071aba2783222132a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:18:13 +0600
Subject: [PATCH 0381/2145] [prosiebensat1] Add title regex

---
 youtube_dl/extractor/prosiebensat1.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 6b13eb605..536a42dc8 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -177,6 +177,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<header class="clearfix">\s*<h3>(.+?)</h3>',
         r'<!-- start video -->\s*<h1>(.+?)</h1>',
         r'<h1 class="att-name">\s*(.+?)</h1>',
+        r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',

From 9f4323252abade4f10b0884682f92cedc78b4d4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:56:50 +0600
Subject: [PATCH 0382/2145] [YoutubeDL] Fix for multiple URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4b801a917..8dbad7cf8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -283,7 +283,6 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
-        self._force_generic_extractor_required = params.get('force_generic_extractor', False)
         self.params = params
         self.cache = Cache(self)
 
@@ -1504,6 +1503,7 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
+                self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
                 # It also downloads the videos
                 res = self.extract_info(url)
             except UnavailableVideoError:

From 61aa5ba36eea3b7cf8c3570ab33604dd2c13b855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 02:05:21 +0600
Subject: [PATCH 0383/2145] [YoutubeDL] Remove global state for
 force_generic_extractor flag in favor of passing argument

---
 youtube_dl/YoutubeDL.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8dbad7cf8..dd2d8cb3c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -627,15 +627,14 @@ class YoutubeDL(object):
             info_dict.setdefault(key, value)
 
     def extract_info(self, url, download=True, ie_key=None, extra_info={},
-                     process=True):
+                     process=True, force_generic_extractor=False):
         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result
         '''
 
-        if not ie_key and self._force_generic_extractor_required:
-            self._force_generic_extractor_required = False
+        if not ie_key and force_generic_extractor:
             ie_key = 'Generic'
 
         if ie_key:
@@ -663,7 +662,7 @@ class YoutubeDL(object):
                     }
                 self.add_default_extra_info(ie_result, ie, url)
                 if process:
-                    return self.process_ie_result(ie_result, download, extra_info)
+                    return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
@@ -688,7 +687,7 @@ class YoutubeDL(object):
             'extractor_key': ie.ie_key(),
         })
 
-    def process_ie_result(self, ie_result, download=True, extra_info={}):
+    def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -716,7 +715,8 @@ class YoutubeDL(object):
             return self.extract_info(ie_result['url'],
                                      download,
                                      ie_key=ie_result.get('ie_key'),
-                                     extra_info=extra_info)
+                                     extra_info=extra_info,
+                                     force_generic_extractor=force_generic_extractor)
         elif result_type == 'url_transparent':
             # Use the information from the embedding page
             info = self.extract_info(
@@ -1503,9 +1503,9 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
-                self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
                 # It also downloads the videos
-                res = self.extract_info(url)
+                res = self.extract_info(
+                    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
             except UnavailableVideoError:
                 self.report_error('unable to download video')
             except MaxDownloadsReached:

From 0072afca8e02052c77dc3b7009e51114887e31b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 02:21:29 +0600
Subject: [PATCH 0384/2145] [YoutubeDL] Remove force_generic_extractor arg from
 process_ie_result

---
 youtube_dl/YoutubeDL.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dd2d8cb3c..a7d3a1c01 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -662,7 +662,7 @@ class YoutubeDL(object):
                     }
                 self.add_default_extra_info(ie_result, ie, url)
                 if process:
-                    return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
+                    return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
@@ -687,7 +687,7 @@ class YoutubeDL(object):
             'extractor_key': ie.ie_key(),
         })
 
-    def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
+    def process_ie_result(self, ie_result, download=True, extra_info={}):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -715,8 +715,7 @@ class YoutubeDL(object):
             return self.extract_info(ie_result['url'],
                                      download,
                                      ie_key=ie_result.get('ie_key'),
-                                     extra_info=extra_info,
-                                     force_generic_extractor=force_generic_extractor)
+                                     extra_info=extra_info)
         elif result_type == 'url_transparent':
             # Use the information from the embedding page
             info = self.extract_info(

From 4f3bf679f5a764f7a26d3c45c82be43e34a3cc4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:09:35 +0600
Subject: [PATCH 0385/2145] [vk] Fix authentication for non-ASCII
 login/password

---
 youtube_dl/extractor/vk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index cc384adbf..d0e772108 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -119,8 +119,8 @@ class VKIE(InfoExtractor):
             'act': 'login',
             'role': 'al_frame',
             'expire': '1',
-            'email': username,
-            'pass': password,
+            'email': username.encode('cp1251'),
+            'pass': password.encode('cp1251'),
         }
 
         request = compat_urllib_request.Request('https://login.vk.com/?act=login',

From 9fcbd5db2abb5a56beefe4a64486da692705ad12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:24:36 +0600
Subject: [PATCH 0386/2145] [pornhub] Add support for embeds

---
 youtube_dl/extractor/pornhub.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index daa284ea2..3c99b4def 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -19,7 +19,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-f]+)'
     _TEST = {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
@@ -39,7 +39,8 @@ class PornHubIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = compat_urllib_request.Request(
+            'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 65d161c480e9964026e618a2e95f9fc9eb8119e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:36:16 +0600
Subject: [PATCH 0387/2145] [extractor/generic] Add support for pornhub embeds

---
 youtube_dl/extractor/generic.py | 5 +++++
 youtube_dl/extractor/pornhub.py | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40d869c53..f683760e4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -42,6 +42,7 @@ from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
+from .pornhub import PornHubIE
 
 
 class GenericIE(InfoExtractor):
@@ -1321,6 +1322,10 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        pornhub_url = PornHubIE._extract_url(webpage)
+        if pornhub_url:
+            return self.url_result(pornhub_url, 'PornHub')
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3c99b4def..8565d7551 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -32,6 +32,13 @@ class PornHubIE(InfoExtractor):
         }
     }
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?pornhub\.com/embed/\d+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _extract_count(self, pattern, webpage, name):
         return str_to_int(self._search_regex(
             pattern, webpage, '%s count' % name, fatal=False))

From 78e2b74bb945dc7f1724f7486405dd523486d634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:39:14 +0600
Subject: [PATCH 0388/2145] [tumblr] Add support for pornhub embeds (Closes
 #5963)

---
 youtube_dl/extractor/generic.py | 1 +
 youtube_dl/extractor/tumblr.py  | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f683760e4..f6b984300 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1322,6 +1322,7 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        # Look for embedded PornHub player
         pornhub_url = PornHubIE._extract_url(webpage)
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index e6218808f..63c20310d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .pornhub import PornHubIE
 
 
 class TumblrIE(InfoExtractor):
@@ -55,6 +56,10 @@ class TumblrIE(InfoExtractor):
         if vid_me_embed_url is not None:
             return self.url_result(vid_me_embed_url, 'Vidme')
 
+        pornhub_url = PornHubIE._extract_url(webpage)
+        if pornhub_url:
+            return self.url_result(pornhub_url, 'PornHub')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From b4e1576aee7cf18f5842714c87985ae0b72f1546 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 13 Jun 2015 06:09:44 -0500
Subject: [PATCH 0389/2145] Brightcove extractor: support
 customBC.createVideo(...); method found in
 http://www.americanbar.org/groups/family_law.html and
 http://america.aljazeera.com/watch/shows/america-tonight/2015/6/exclusive-hunting-isil-with-the-pkk.html

---
 youtube_dl/extractor/brightcove.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c1d4320e1..20a6ed965 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -188,7 +188,19 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
+        if matches:
+            return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
+
+        custombcs = re.findall(r'customBC.\createVideo\((.+?)\);',webpage)
+        if custombcs:
+            urls = []
+            for match in custombcs:
+                # brightcove playerkey begins with AQ and is 50 characters in length,
+                # however it's appended to itself in places, so truncate.
+                f = re.search(r'["\'](AQ[^"\']{48}).*?["\'](\d+)["\']', match)
+                if f:
+                    urls.append('brightcove:playerKey='+f.group(1)+'&%40videoPlayer='+f.group(2))
+            return urls
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From af9cdee9cba610aa3924f90a8a3fcd7dd43c65eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 19:53:32 +0600
Subject: [PATCH 0390/2145] [brightcove] Improve and generalize brightcove URL
 extraction from JS

---
 youtube_dl/extractor/brightcove.py | 35 +++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 20a6ed965..d768f99e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -156,6 +156,28 @@ class BrightcoveIE(InfoExtractor):
         linkBase = find_param('linkBaseURL')
         if linkBase is not None:
             params['linkBaseURL'] = linkBase
+        return cls._make_brightcove_url(params)
+
+    @classmethod
+    def _build_brighcove_url_from_js(cls, object_js):
+        # The layout of JS is as follows:
+        # customBC.createVideo = function (width, height, playerID, playerKey, videoPlayer, VideoRandomID) {
+        #   // build Brightcove <object /> XML
+        # }
+        m = re.search(
+            r'''(?x)customBC.\createVideo\(
+                .*?                                                  # skipping width and height
+                ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
+                ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters
+                                                                     # in length, however it's appended to itself
+                                                                     # in places, so truncate
+                ["\'](?P<videoID>\d+)["\']                           # @videoPlayer
+            ''', object_js)
+        if m:
+            return cls._make_brightcove_url(m.groupdict())
+
+    @classmethod
+    def _make_brightcove_url(cls, params):
         data = compat_urllib_parse.urlencode(params)
         return cls._FEDERATED_URL_TEMPLATE % data
 
@@ -191,16 +213,9 @@ class BrightcoveIE(InfoExtractor):
         if matches:
             return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
-        custombcs = re.findall(r'customBC.\createVideo\((.+?)\);',webpage)
-        if custombcs:
-            urls = []
-            for match in custombcs:
-                # brightcove playerkey begins with AQ and is 50 characters in length,
-                # however it's appended to itself in places, so truncate.
-                f = re.search(r'["\'](AQ[^"\']{48}).*?["\'](\d+)["\']', match)
-                if f:
-                    urls.append('brightcove:playerKey='+f.group(1)+'&%40videoPlayer='+f.group(2))
-            return urls
+        return list(filter(None, [
+            cls._build_brighcove_url_from_js(custom_bc)
+            for custom_bc in re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)]))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From 0029071adbdc0e1469d76cdc7e058c2f77299610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 07:43:14 +0600
Subject: [PATCH 0391/2145] [dramefever] Improve and simplify

---
 youtube_dl/extractor/dramafever.py | 172 +++++++++++++++++------------
 1 file changed, 101 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 40787ffcd..0f33a61a3 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -1,104 +1,111 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
+import itertools
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_HTTPError,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class DramaFeverIE(InfoExtractor):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)/'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)'
+    _TEST = {
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
             'ext': 'flv',
             'title': 'Cooking with Shin 4512.1',
+            'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1404336058,
             'upload_date': '20140702',
-            'description': 'Served at all special occasions and featured in the hit drama Heirs, Shin cooks Red Bean Rice.',
+            'duration': 343,
         }
-    }]
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url).replace("/", ".")
+        video_id = self._match_id(url).replace('/', '.')
 
-        consumer_secret = self._get_consumer_secret(video_id)
+        try:
+            feed = self._download_json(
+                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
+                video_id, 'Downloading episode JSON')['channel']['item']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                raise ExtractorError(
+                    'Currently unavailable in your country.', expected=True)
+            raise
 
-        ep_json = self._download_json(
-            "http://www.dramafever.com/amp/episode/feed.json?guid=%s" % video_id,
-            video_id, note='Downloading episode metadata',
-            errnote="Video may not be available for your location")["channel"]["item"]
-
-        title = ep_json["media-group"]["media-title"]
-        description = ep_json["media-group"]["media-description"]
-        thumbnail = ep_json["media-group"]["media-thumbnail"]["@attributes"]["url"]
-        duration = int(ep_json["media-group"]["media-content"][0]["@attributes"]["duration"])
-        mobj = re.match(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", ep_json["pubDate"])
-        upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) if mobj is not None else None
+        media_group = feed.get('media-group', {})
 
         formats = []
-        for vid_format in ep_json["media-group"]["media-content"]:
-            src = vid_format["@attributes"]["url"]
-            if '.f4m' in src:
-                formats.extend(self._extract_f4m_formats(src, video_id))
-
+        for media_content in media_group['media-content']:
+            src = media_content.get('@attributes', {}).get('url')
+            if not src:
+                continue
+            ext = determine_ext(src)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    src, video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': src,
+                })
         self._sort_formats(formats)
-        video_subtitles = self.extract_subtitles(video_id, consumer_secret)
+
+        title = media_group.get('media-title')
+        description = media_group.get('media-description')
+        duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
+        thumbnail = self._proto_relative_url(
+            media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
+        timestamp = parse_iso8601(feed.get('pubDate'), ' ')
+
+        subtitles = {}
+        for media_subtitle in media_group.get('media-subTitle', []):
+            lang = media_subtitle.get('@attributes', {}).get('lang')
+            href = media_subtitle.get('@attributes', {}).get('href')
+            if not lang or not href:
+                continue
+            subtitles[lang] = [{
+                'ext': 'ttml',
+                'url': href,
+            }]
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
-    def _get_consumer_secret(self, video_id):
-        df_js = self._download_webpage(
-            "http://www.dramafever.com/static/126960d/v2/js/plugins/jquery.threadedcomments.js", video_id)
-        return self._search_regex(r"'cs': '([0-9a-zA-Z]+)'", df_js, "cs")
 
-    def _get_episodes(self, series_id, consumer_secret, episode_filter=None):
-        _PAGE_SIZE = 60
-
-        curr_page = 1
-        max_pages = curr_page + 1
-        results = []
-        while max_pages >= curr_page:
-            page_url = "http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d" % \
-                       (consumer_secret, series_id, _PAGE_SIZE, curr_page)
-            series = self._download_json(
-                page_url, series_id, note="Downloading series json page #%d" % curr_page)
-            max_pages = series['num_pages']
-            results.extend([ep for ep in series['value'] if episode_filter is None or episode_filter(ep)])
-            curr_page += 1
-        return results
-
-    def _get_subtitles(self, video_id, consumer_secret):
-
-        res = None
-        info = self._get_episodes(
-            video_id.split(".")[0], consumer_secret,
-            episode_filter=lambda x: x['guid'] == video_id)
-
-        if len(info) == 1 and info[0]['subfile'] != '':
-            res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
-        return res
-
-
-class DramaFeverSeriesIE(DramaFeverIE):
+class DramaFeverSeriesIE(InfoExtractor):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)/\d*[a-zA-Z_][a-zA-Z0-9_]*/'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512',
             'title': 'Cooking with Shin',
-            'description': 'Professional chef and cooking instructor Shin Kim takes some of the delicious dishes featured in your favorite dramas and shows you how to make them right at home.',
+            'description': 'md5:84a3f26e3cdc3fb7f500211b3593b5c1',
         },
         'playlist_count': 4,
     }, {
@@ -106,25 +113,48 @@ class DramaFeverSeriesIE(DramaFeverIE):
         'info_dict': {
             'id': '124',
             'title': 'IRIS',
-            'description': 'Lee Byung Hun and Kim Tae Hee star in this powerhouse drama and ratings megahit of action, intrigue and romance.',
+            'description': 'md5:b3a30e587cf20c59bd1c01ec0ee1b862',
         },
         'playlist_count': 20,
     }]
 
+    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
+    _PAGE_SIZE = 5  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
+
+    def _get_consumer_secret(self, video_id):
+        mainjs = self._download_webpage(
+            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
+            video_id, 'Downloading main.js', fatal=False)
+        if not mainjs:
+            return self._CONSUMER_SECRET
+        return self._search_regex(
+            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
+            'consumer secret', default=self._CONSUMER_SECRET)
+
     def _real_extract(self, url):
         series_id = self._match_id(url)
+
         consumer_secret = self._get_consumer_secret(series_id)
 
-        series_json = self._download_json(
-            "http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s" % (consumer_secret, series_id),
-            series_id, note='Downloading series metadata')["series"][series_id]
+        series = self._download_json(
+            'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
+            % (consumer_secret, series_id),
+            series_id, 'Downloading series JSON')['series'][series_id]
 
-        title = series_json["name"]
-        description = series_json["description_short"]
+        title = clean_html(series['name'])
+        description = clean_html(series.get('description') or series.get('description_short'))
 
-        episodes = self._get_episodes(series_id, consumer_secret)
         entries = []
-        for ep in episodes:
-            entries.append(self.url_result(
-                'http://www.dramafever.com%s' % ep['episode_url'], 'DramaFever', ep['guid']))
+        for page_num in itertools.count(1):
+            episodes = self._download_json(
+                'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
+                % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
+                series_id, 'Downloading episodes JSON page #%d' % page_num)
+            for episode in episodes.get('value', []):
+                entries.append(self.url_result(
+                    compat_urlparse.urljoin(url, episode['episode_url']),
+                    'DramaFever', episode.get('guid')))
+            if page_num == episodes['num_pages']:
+                break
+
         return self.playlist_result(entries, series_id, title, description)

From 70a2002399b46aa0cde2879d856d1bb68e2c6f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:50:23 +0600
Subject: [PATCH 0392/2145] [dramafever:series] Fix _VALID_URL (Closes #5973)

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 0f33a61a3..42e0df24e 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -99,7 +99,7 @@ class DramaFeverIE(InfoExtractor):
 
 class DramaFeverSeriesIE(InfoExtractor):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {

From 463b2e5542a85d5cd41b255a71833fec7b4f51e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:51:07 +0600
Subject: [PATCH 0393/2145] [dramafever:series] Rollback _PAGE_SIZE to max
 possible

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 42e0df24e..c4b7c0b68 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -119,7 +119,7 @@ class DramaFeverSeriesIE(InfoExtractor):
     }]
 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
-    _PAGE_SIZE = 5  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
+    _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 
     def _get_consumer_secret(self, video_id):
         mainjs = self._download_webpage(

From 450d89ddc12d80a500a2429632d35a0045cf630b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:58:26 +0600
Subject: [PATCH 0394/2145] [dramafever] Improve _VALID_URL

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index c4b7c0b68..a34aad486 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 class DramaFeverIE(InfoExtractor):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TEST = {
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {

From 976b03c56bddf20c978820474e307457523f4c05 Mon Sep 17 00:00:00 2001
From: chaoskagami <chaos.kagami@gmail.com>
Date: Sun, 14 Jun 2015 00:18:40 -0400
Subject: [PATCH 0395/2145] Quality note for niconico - at least notify whether
 you'll get low or src

---
 youtube_dl/extractor/niconico.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 3cecebf95..e10348004 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -184,6 +184,11 @@ class NiconicoIE(InfoExtractor):
             extension = determine_ext(video_real_url)
         video_format = extension.upper()
 
+        if video_real_url.endswith('low'):
+            format_note = 'low'
+        else:
+            format_note = 'src'
+
         thumbnail = (
             xpath_text(video_info, './/thumbnail_url') or
             self._html_search_meta('image', webpage, 'thumbnail', default=None) or
@@ -242,6 +247,7 @@ class NiconicoIE(InfoExtractor):
             'title': title,
             'ext': extension,
             'format': video_format,
+            'format_note' : format_note,
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 180940e02df60129bce36035b4a2fd79c0c60995 Mon Sep 17 00:00:00 2001
From: Shrimadhav U K <SpEcHiDe@users.noreply.github.com>
Date: Sun, 14 Jun 2015 11:19:42 +0530
Subject: [PATCH 0396/2145] spelling mistake corrected

acces changed to accessing
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..aacec2958 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -119,7 +119,7 @@ class YoutubeDL(object):
 
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
-    videopassword:     Password for acces a video.
+    videopassword:     Password for accessing a video.
     usenetrc:          Use netrc for authentication instead.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.

From 755a9d3d1a8f99b061c8d29525d629b8ad6061a4 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Jun 2015 20:58:15 +0300
Subject: [PATCH 0397/2145] [tvplay] Add support for NovaTv

---
 youtube_dl/extractor/tvplay.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index e83e31a31..79863e781 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -26,6 +26,7 @@ class TVPlayIE(InfoExtractor):
            viasat4play\.no/programmer|
            tv6play\.no/programmer|
            tv3play\.dk/programmer|
+           play\.novatv\.bg/programi
         )/[^/]+/(?P<id>\d+)
         '''
     _TESTS = [
@@ -173,6 +174,22 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
+            'info_dict': {
+                'id': '624952',
+                'ext': 'flv',
+                'title': 'Здравей, България (12.06.2015 г.) ',
+                'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
+                'duration': 8838,
+                'timestamp': 1434100372,
+                'upload_date': '20150612',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):

From 9fd24e3a227a059eed07b679dac858e5bd747123 Mon Sep 17 00:00:00 2001
From: jomo <github@jomo.tv>
Date: Sun, 14 Jun 2015 21:50:03 +0200
Subject: [PATCH 0398/2145] LiveLeak: support more original videos

some (old?) videos use ...mp4.h264_270p.mp4... instead of ...mp4.h264_base.mp4...
This is an addition to #4768
---
 youtube_dl/extractor/liveleak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 35822067f..431f2e85d 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -85,7 +85,7 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
-            orig_url = s['file'].replace('.h264_base.mp4', '')
+            orig_url = re.sub(r'.h264_.+\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({
                     'format_id': 'original-%s' % i,

From 8f75761f24f9f2599efe100b5a094182af6403d0 Mon Sep 17 00:00:00 2001
From: jomo <github@jomo.tv>
Date: Sun, 14 Jun 2015 22:41:44 +0200
Subject: [PATCH 0399/2145] LiveLak: add test for URLs with 'h264_270p'

---
 youtube_dl/extractor/liveleak.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 431f2e85d..c658cc92b 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -40,6 +40,16 @@ class LiveLeakIE(InfoExtractor):
             'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
             'age_limit': 18,
         }
+    }, {
+        'url': 'http://www.liveleak.com/view?i=801_1409392012',
+        'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
+        'info_dict': {
+            'id': '801_1409392012',
+            'ext': 'mp4',
+            'description': "Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.",
+            'uploader': 'bony333',
+            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia'
+        }
     }]
 
     def _real_extract(self, url):

From 00ac23e6e06bf6de59d5d5b3f42ff64ce039fee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:51:21 +0600
Subject: [PATCH 0400/2145] [liveleak] Improve regex for restoring original
 video URL

---
 youtube_dl/extractor/liveleak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c658cc92b..e82f21ea7 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -95,7 +95,7 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
-            orig_url = re.sub(r'.h264_.+\.mp4', '', s['file'])
+            orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({
                     'format_id': 'original-%s' % i,

From afa1ded425ffe12a5b90bcd4d316c43941a5dc1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:54:05 +0600
Subject: [PATCH 0401/2145] [liveleak] Clarify rationale for restoring raw
 video

---
 youtube_dl/extractor/liveleak.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index e82f21ea7..0a4e473d6 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -95,6 +95,9 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
+            # Removing '.h264_*.mp4' gives the raw video, which is essentially
+            # the same video without the LiveLeak logo at the top (see
+            # https://github.com/rg3/youtube-dl/pull/4768)
             orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({

From b95cfa917025750805fb873fc4e2eb161241b22b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:54:49 +0600
Subject: [PATCH 0402/2145] [liveleak] Clarify test

---
 youtube_dl/extractor/liveleak.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 0a4e473d6..857edfde2 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -41,6 +41,7 @@ class LiveLeakIE(InfoExtractor):
             'age_limit': 18,
         }
     }, {
+        # Covers https://github.com/rg3/youtube-dl/pull/5983
         'url': 'http://www.liveleak.com/view?i=801_1409392012',
         'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
         'info_dict': {

From 5774ef35c4d167f7c959041bf4efc5581a98f0a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:57:07 +0600
Subject: [PATCH 0403/2145] [options] Add missing whitespace for --fixup
 description

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..740458e51 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -725,7 +725,7 @@ def parseOpts(overrideArguments=None):
         metavar='POLICY', dest='fixup', default='detect_or_warn',
         help='Automatically correct known faults of the file. '
              'One of never (do nothing), warn (only emit a warning), '
-             'detect_or_warn(the default; fix file if we can, warn otherwise)')
+             'detect_or_warn (the default; fix file if we can, warn otherwise)')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',

From 67d95f177c7ffedfc8f8b086535013a1a7a48b29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 03:43:33 +0600
Subject: [PATCH 0404/2145] [niconico] Simplify format info

---
 youtube_dl/extractor/niconico.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index e10348004..0f8aa5ada 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -182,12 +182,6 @@ class NiconicoIE(InfoExtractor):
         extension = xpath_text(video_info, './/movie_type')
         if not extension:
             extension = determine_ext(video_real_url)
-        video_format = extension.upper()
-
-        if video_real_url.endswith('low'):
-            format_note = 'low'
-        else:
-            format_note = 'src'
 
         thumbnail = (
             xpath_text(video_info, './/thumbnail_url') or
@@ -246,8 +240,7 @@ class NiconicoIE(InfoExtractor):
             'url': video_real_url,
             'title': title,
             'ext': extension,
-            'format': video_format,
-            'format_note' : format_note,
+            'format_id': 'economy' if video_real_url.endswith('low') else 'normal',
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 2a0fcf6113c3f0c8d0510167fd7017cc0fdfa622 Mon Sep 17 00:00:00 2001
From: zx8 <zx8@mac>
Date: Mon, 15 Jun 2015 00:27:43 +0100
Subject: [PATCH 0405/2145] [safari] make url regex more lenient

---
 youtube_dl/extractor/safari.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 10251f29e..20ba6fa33 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -83,7 +83,7 @@ class SafariIE(SafariBaseIE):
                                     library/view/[^/]+|
                                     api/v1/book
                                 )/
-                                (?P<course_id>\d+)/
+                                (?P<course_id>[^/]+)/
                                     (?:chapter(?:-content)?/)?
                                 (?P<part>part\d+)\.html
     '''
@@ -122,7 +122,7 @@ class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
 
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>\d+)/?(?:[#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>[^/]+)/?(?:[#?]|$)'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',

From 4b9f9010b0d744969189c383e98e8729f9fe9623 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Jun 2015 01:35:50 +0200
Subject: [PATCH 0406/2145] release 2015.06.15

---
 README.md              | 4 ++--
 docs/supportedsites.md | 7 +++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index f3d83c89f..5f3a08f5a 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ which means you can modify it, redistribute it or use it however you like.
     -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
     --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
     --dump-user-agent                Display the current browser identification
-    --list-extractors                List all supported extractors and the URLs they would handle
+    --list-extractors                List all supported extractors
     --extractor-descriptions         Output descriptions of all supported extractors
     --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
@@ -223,7 +223,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
+    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the default;
                                      fix file if we can, warn otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d147b53fe..220e52b98 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -120,6 +120,8 @@
  - **divxstage**: DivxStage
  - **Dotsub**
  - **DouyuTV**
+ - **dramafever**
+ - **dramafever:series**
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
@@ -153,6 +155,7 @@
  - **fernsehkritik.tv**
  - **fernsehkritik.tv:postecke**
  - **Firstpost**
+ - **FiveTV**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
@@ -217,6 +220,7 @@
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
  - **IPrima**
+ - **iqiyi**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -407,6 +411,7 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **Ruutu**
  - **safari**: safaribooksonline.com online video
  - **safari:course**: safaribooksonline.com online courses
  - **Sandia**: Sandia National Laboratories
@@ -519,6 +524,8 @@
  - **TV2**
  - **TV2Article**
  - **TV4**: tv4.se and tv4play.se
+ - **TVC**
+ - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
  - **tvp.pl:Series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9cf84ff71..34a13cb81 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.04.1'
+__version__ = '2015.06.15'

From 4af98ecdfb896c16e73cb9f7306908cc686782e8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 18:49:27 +0800
Subject: [PATCH 0407/2145] [vbox7] Fix extraction (fixes #5967)

---
 youtube_dl/extractor/vbox7.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index dd026748d..722eb5236 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -26,11 +27,21 @@ class Vbox7IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        redirect_page, urlh = self._download_webpage_handle(url, video_id)
-        new_location = self._search_regex(r'window\.location = \'(.*)\';',
-                                          redirect_page, 'redirect location')
-        redirect_url = urlh.geturl() + new_location
-        webpage = self._download_webpage(redirect_url, video_id,
+        # need to get the page 3 times for the correct jsSecretToken cookie
+        # which is necessary for the correct title
+        def get_session_id():
+            redirect_page = self._download_webpage(url, video_id)
+            session_id_url = self._search_regex(
+                r'var\s*url\s*=\s*\'([^\']+)\';', redirect_page,
+                'session id url')
+            self._download_webpage(
+                compat_urlparse.urljoin(url, session_id_url), video_id,
+                'Getting session id')
+
+        get_session_id()
+        get_session_id()
+
+        webpage = self._download_webpage(url, video_id,
                                          'Downloading redirect page')
 
         title = self._html_search_regex(r'<title>(.*)</title>',

From aed473ccf9d9da73b1b80ee8b06d00ee66a3769d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 22:41:24 +0800
Subject: [PATCH 0408/2145] [youku] PEP8

---
 youtube_dl/extractor/youku.py | 80 +++++++++++++++++------------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index aed6b960a..4e47fca8a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -11,6 +11,7 @@ from ..compat import compat_urllib_parse
 
 bytes_is_str = (bytes == str)  # for compatible
 
+
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
@@ -21,13 +22,13 @@ class YoukuIE(InfoExtractor):
     '''
 
     _TEST = {
-            'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
-            'md5': '5f3af4192eabacc4501508d54a8cabd7',
-            'info_dict': {
-                'id': 'XMTc1ODE5Njcy',
-                'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
-                'ext': 'flv'
-            }
+        'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
+        'md5': '5f3af4192eabacc4501508d54a8cabd7',
+        'info_dict': {
+            'id': 'XMTc1ODE5Njcy',
+            'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
+            'ext': 'flv'
+        }
     }
 
     def construct_video_urls(self, data1, data2):
@@ -36,7 +37,7 @@ class YoukuIE(InfoExtractor):
             ls = list(range(256))
             t = 0
             for i in range(256):
-                t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
+                t = (t + ls[i] + ord(s1[i % len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
             s = '' if not bytes_is_str else b''
             x, y = 0, 0
@@ -45,16 +46,16 @@ class YoukuIE(InfoExtractor):
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
                 if isinstance(s2[i], int):
-                    s += chr(s2[i] ^ ls[(ls[x]+ls[y]) % 256])
+                    s += chr(s2[i] ^ ls[(ls[x] + ls[y]) % 256])
                 else:
-                    s += chr(ord(s2[i]) ^ ls[(ls[x]+ls[y]) % 256])
+                    s += chr(ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
             return s
 
         sid, token = yk_t(
             'becaf9be',
-            base64.b64decode(bytes(data2['ep'], 'ascii')) \
-                if not bytes_is_str \
-                else base64.b64decode(data2['ep'])
+            base64.b64decode(bytes(data2['ep'], 'ascii'))
+            if not bytes_is_str
+            else base64.b64decode(data2['ep'])
         ).split('_')
 
         # get oip
@@ -89,13 +90,13 @@ class YoukuIE(InfoExtractor):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
                 'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
-                if not bytes_is_str \
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii')
+                if not bytes_is_str
                 else ('%s_%s_%s' % (sid, fileid, token))
             )
             ep = base64.b64encode(
-                bytes(ep_t, 'latin') \
-                if not bytes_is_str \
+                bytes(ep_t, 'latin')
+                if not bytes_is_str
                 else ep_t
             ).decode()
             return ep
@@ -121,9 +122,9 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_' + str(int(n)+1).zfill(2) + \
+                    '_' + str(int(n) + 1).zfill(2) + \
                     '/st/' + self.parse_ext_l(format) + \
-                    '/fileid/' + get_fileid(format, n)  + '?' + \
+                    '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
@@ -132,34 +133,34 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv'   : '0',
-            'mp4'   : '1',
-            'hd2'   : '2',
-            'hd3'   : '3',
-            '3gp'   : '0',
-            '3gphd' : '1'
+            'flv': '0',
+            'mp4': '1',
+            'hd2': '2',
+            'hd3': '3',
+            '3gp': '0',
+            '3gphd': '1'
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
-            'flv'   : 'flv',
-            'mp4'   : 'mp4',
-            'hd2'   : 'flv',
-            'hd3'   : 'flv',
-            '3gp'   : 'flv',
-            '3gphd' : 'mp4'
+            'flv': 'flv',
+            'mp4': 'mp4',
+            'hd2': 'flv',
+            'hd3': 'flv',
+            '3gp': 'flv',
+            '3gphd': 'mp4'
         }
         return ext_dict[fm]
 
     def get_format_name(self, fm):
         _dict = {
-            '3gp'   : 'h6',
-            '3gphd' : 'h5',
-            'flv'   : 'h4',
-            'mp4'   : 'h3',
-            'hd2'   : 'h2',
-            'hd3'   : 'h1'
+            '3gp': 'h6',
+            '3gphd': 'h5',
+            'flv': 'h4',
+            'mp4': 'h3',
+            'hd2': 'h2',
+            'hd3': 'h1'
         }
         return _dict[fm]
 
@@ -194,10 +195,9 @@ class YoukuIE(InfoExtractor):
         # construct info
         entries = []
         for fm in data1['streamtypes']:
-            #formats = []
             video_urls = video_urls_dict[fm]
             for i in range(len(video_urls)):
-                if len(entries) < i+1:
+                if len(entries) < i + 1:
                     entries.append({'formats': []})
                 entries[i]['formats'].append(
                     {
@@ -211,7 +211,7 @@ class YoukuIE(InfoExtractor):
         for i in range(len(entries)):
             entries[i].update(
                 {
-                    'id': '_part%d' % (i+1),
+                    'id': '_part%d' % (i + 1),
                     'title': title,
                 }
             )

From 054932f4035d606946f0c054c02cf87496b753f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:46:10 +0600
Subject: [PATCH 0409/2145] [vk] Fix extraction (Closes #5987)

---
 youtube_dl/extractor/vk.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d0e772108..6aeba109d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -175,16 +175,16 @@ class VKIE(InfoExtractor):
                 m_rutube.group(1).replace('\\', ''))
             return self.url_result(rutube_url)
 
-        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.*?});', info_page)
+        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
         if m_opts:
-            m_opts_url = re.search(r"url\s*:\s*'([^']+)", m_opts.group(1))
+            m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))
             if m_opts_url:
                 opts_url = m_opts_url.group(1)
                 if opts_url.startswith('//'):
                     opts_url = 'http:' + opts_url
                 return self.url_result(opts_url)
 
-        data_json = self._search_regex(r'var vars = ({.*?});', info_page, 'vars')
+        data_json = self._search_regex(r'var\s+vars\s*=\s*({.+?});', info_page, 'vars')
         data = json.loads(data_json)
 
         # Extract upload date

From 7c7dd9dc7fe8139196c7bd1c512301a61f9f362b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:47:01 +0600
Subject: [PATCH 0410/2145] [vk] Fix upload date extraction

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 6aeba109d..f974f8fef 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -189,7 +189,7 @@ class VKIE(InfoExtractor):
 
         # Extract upload date
         upload_date = None
-        mobj = re.search(r'id="mv_date_wrap".*?Added ([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
+        mobj = re.search(r'id="mv_date(?:_views)?_wrap"[^>]*>([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
         if mobj is not None:
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))

From 8117df4cd9e49a3c7369db3cab6c0b94365c7786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:55:25 +0600
Subject: [PATCH 0411/2145] [vk] Extract view count

---
 youtube_dl/extractor/vk.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f974f8fef..38ff3c1a9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -13,6 +13,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     orderedSet,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
 )
@@ -34,6 +35,7 @@ class VKIE(InfoExtractor):
                 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                 'duration': 195,
                 'upload_date': '20120212',
+                'view_count': int,
             },
         },
         {
@@ -45,7 +47,8 @@ class VKIE(InfoExtractor):
                 'uploader': 'Tom Cruise',
                 'title': 'No name',
                 'duration': 9,
-                'upload_date': '20130721'
+                'upload_date': '20130721',
+                'view_count': int,
             }
         },
         {
@@ -59,6 +62,7 @@ class VKIE(InfoExtractor):
                 'title': 'Lin Dan',
                 'duration': 101,
                 'upload_date': '20120730',
+                'view_count': int,
             }
         },
         {
@@ -73,7 +77,8 @@ class VKIE(InfoExtractor):
                 'uploader': 'Триллеры',
                 'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
                 'duration': 8352,
-                'upload_date': '20121218'
+                'upload_date': '20121218',
+                'view_count': int,
             },
             'skip': 'Requires vk account credentials',
         },
@@ -100,6 +105,7 @@ class VKIE(InfoExtractor):
                 'title': 'Книга Илая',
                 'duration': 6771,
                 'upload_date': '20140626',
+                'view_count': int,
             },
             'skip': 'Only works from Russia',
         },
@@ -194,6 +200,10 @@ class VKIE(InfoExtractor):
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
+        view_count = str_to_int(self._search_regex(
+            r'"mv_views_count_number"[^>]*>([\d,.]+) views<',
+            info_page, 'view count', fatal=False))
+
         formats = [{
             'format_id': k,
             'url': v,
@@ -210,6 +220,7 @@ class VKIE(InfoExtractor):
             'uploader': data.get('md_author'),
             'duration': data.get('duration'),
             'upload_date': upload_date,
+            'view_count': view_count,
         }
 
 

From 02175a7986c4223a0ed27a872c1ca16926913e05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:01:26 +0600
Subject: [PATCH 0412/2145] [youtube:search] Fix search query (Closes #5988)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3448bec4f..9e2671192 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1504,7 +1504,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
 
         for pagenum in itertools.count(1):
             url_query = {
-                'search_query': query,
+                'search_query': query.encode('utf-8'),
                 'page': pagenum,
                 'spf': 'navigate',
             }

From c203be3fb4f00388c81564dc0c85ff8a10ff4553 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:28:59 +0800
Subject: [PATCH 0413/2145] [youku] Better handling for Python 2/3
 compatibility

---
 youtube_dl/extractor/youku.py | 37 ++++++++++++-----------------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 4e47fca8a..26e5baadc 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -7,9 +7,10 @@ import base64
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
-from ..compat import compat_urllib_parse
-
-bytes_is_str = (bytes == str)  # for compatible
+from ..compat import (
+    compat_urllib_parse,
+    compat_ord,
+)
 
 
 class YoukuIE(InfoExtractor):
@@ -37,26 +38,20 @@ class YoukuIE(InfoExtractor):
             ls = list(range(256))
             t = 0
             for i in range(256):
-                t = (t + ls[i] + ord(s1[i % len(s1)])) % 256
+                t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s = '' if not bytes_is_str else b''
+            s = bytearray()
             x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
-                if isinstance(s2[i], int):
-                    s += chr(s2[i] ^ ls[(ls[x] + ls[y]) % 256])
-                else:
-                    s += chr(ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
-            return s
+                s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
+            return bytes(s)
 
         sid, token = yk_t(
-            'becaf9be',
-            base64.b64decode(bytes(data2['ep'], 'ascii'))
-            if not bytes_is_str
-            else base64.b64decode(data2['ep'])
-        ).split('_')
+            b'becaf9be', base64.b64decode(data2['ep'].encode('ascii'))
+        ).decode('ascii').split('_')
 
         # get oip
         oip = data2['ip']
@@ -89,16 +84,10 @@ class YoukuIE(InfoExtractor):
         def generate_ep(format, n):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
-                'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii')
-                if not bytes_is_str
-                else ('%s_%s_%s' % (sid, fileid, token))
+                b'bf7e5f01',
+                ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
             )
-            ep = base64.b64encode(
-                bytes(ep_t, 'latin')
-                if not bytes_is_str
-                else ep_t
-            ).decode()
+            ep = base64.b64encode(ep_t).decode('ascii')
             return ep
 
         # generate video_urls

From 99e6833c85868b78df7c810603ffdccdaeb4eaf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:30:27 +0600
Subject: [PATCH 0414/2145] [francetv] Update f4m manifest token URL (Closes
 #5981, Closes #5989)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index edf555b29..db0bbec1e 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -60,7 +60,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     continue
                 video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
-                    'http://hdfauth.francetv.fr/esi/urltokengen2.html?url=%s' % video_url_parsed.path,
+                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url_parsed.path,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))

From 9383e66f9475eca0e64c09972c1392d92d17570c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:31:30 +0800
Subject: [PATCH 0415/2145] [youku] Use _match_id

---
 youtube_dl/extractor/youku.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 26e5baadc..e41b48369 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
@@ -154,8 +153,7 @@ class YoukuIE(InfoExtractor):
         return _dict[fm]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # request basic data
         data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id

From ee69799262e8344742b9d8b492fe792b4d586f6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:36:28 +0800
Subject: [PATCH 0416/2145] [youku] Add a v.swf test case

---
 youtube_dl/extractor/youku.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e41b48369..d8162a0c5 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -21,7 +21,7 @@ class YoukuIE(InfoExtractor):
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
@@ -29,7 +29,10 @@ class YoukuIE(InfoExtractor):
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'flv'
         }
-    }
+    }, {
+        'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
+        'only_matching': True,
+    }]
 
     def construct_video_urls(self, data1, data2):
         # get sid, token

From 4fd35ee072a39654f11e794db5c50ee375c9a7c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:36:30 +0600
Subject: [PATCH 0417/2145] [safari] Add test for #5985

---
 youtube_dl/extractor/safari.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 20ba6fa33..f3c80708c 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -100,6 +100,10 @@ class SafariIE(SafariBaseIE):
     }, {
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
         'only_matching': True,
+    }, {
+        # non-digits in course id
+        'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f1e66cb2eb40b48c6508acbe57207a2d99792bf0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:46:07 +0800
Subject: [PATCH 0418/2145] [youku] Change video_id and add a multipart test
 case

---
 youtube_dl/extractor/youku.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d8162a0c5..d5b73ebce 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -25,13 +25,20 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
-            'id': 'XMTc1ODE5Njcy',
+            'id': 'XMTc1ODE5Njcy_part1',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'flv'
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
         'only_matching': True,
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
+        'info_dict': {
+            'id': 'XODgxNjg1Mzk2',
+            'title': '武媚娘传奇 85',
+        },
+        'playlist_count': 11,
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -201,20 +208,14 @@ class YoukuIE(InfoExtractor):
         for i in range(len(entries)):
             entries[i].update(
                 {
-                    'id': '_part%d' % (i + 1),
+                    'id': '%s_part%d' % (video_id, i + 1),
                     'title': title,
                 }
             )
 
-        if len(entries) > 1:
-            info = {
-                '_type': 'multi_video',
-                'id': video_id,
-                'title': title,
-                'entries': entries,
-            }
-        else:
-            info = entries[0]
-            info['id'] = video_id
-
-        return info
+        return {
+            '_type': 'multi_video',
+            'id': video_id,
+            'title': title,
+            'entries': entries,
+        }

From 04e7596680bce28beae2436bac0f6d1f01a45210 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:54:55 +0800
Subject: [PATCH 0419/2145] [youku] Better error handling

---
 youtube_dl/extractor/youku.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d5b73ebce..91f9f6bff 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -176,13 +176,15 @@ class YoukuIE(InfoExtractor):
 
         error_code = data1.get('error_code')
         if error_code:
-            # -8 means blocked outside China.
-            # Chinese and English, separated by newline.
             error = data1.get('error')
-            raise ExtractorError(
-                error or 'Server reported error %i' %
-                error_code,
-                expected=True)
+            if error is not None and '因版权原因无法观看此视频' in error:
+                raise ExtractorError(
+                    'Youku said: Sorry, this video is available in China only', expected=True)
+            else:
+                msg = 'Youku server reported error %i' % error_code
+                if error is not None:
+                    msg += ': ' + error
+                raise ExtractorError(msg)
 
         title = data1['title']
 

From 5228b756af2c2bfc2962a5b1bb6db1e6a41c9e05 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:06:23 +0800
Subject: [PATCH 0420/2145] [youku] Add cn_verification_proxy support and add a
 georestricted test case

---
 youtube_dl/extractor/youku.py | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 91f9f6bff..ea37dc8b2 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -9,6 +9,7 @@ from ..utils import ExtractorError
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
+    compat_urllib_request,
 )
 
 
@@ -39,6 +40,14 @@ class YoukuIE(InfoExtractor):
             'title': '武媚娘传奇 85',
         },
         'playlist_count': 11,
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
+        'info_dict': {
+            'id': 'XMTI1OTczNDM5Mg',
+            'title': '花千骨 04',
+        },
+        'playlist_count': 13,
+        'skip': 'Available in China only',
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -165,14 +174,23 @@ class YoukuIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # request basic data
-        data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
-        data2_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id
+        def retrieve_data(req_url, note):
+            req = compat_urllib_request.Request(req_url)
 
-        raw_data1 = self._download_json(data1_url, video_id)
-        raw_data2 = self._download_json(data2_url, video_id)
-        data1 = raw_data1['data'][0]
-        data2 = raw_data2['data'][0]
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
+            raw_data = self._download_json(req, video_id, note=note)
+            return raw_data['data'][0]
+
+        # request basic data
+        data1 = retrieve_data(
+            'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
+            'Downloading JSON metadata 1')
+        data2 = retrieve_data(
+            'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            'Downloading JSON metadata 2')
 
         error_code = data1.get('error_code')
         if error_code:

From a155b7e76c5a71c650f62c4716d23a24943fc373 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:15:09 +0800
Subject: [PATCH 0421/2145] [youku] Coding style

---
 youtube_dl/extractor/youku.py | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ea37dc8b2..cab5be3a4 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -216,22 +216,18 @@ class YoukuIE(InfoExtractor):
             for i in range(len(video_urls)):
                 if len(entries) < i + 1:
                     entries.append({'formats': []})
-                entries[i]['formats'].append(
-                    {
-                        'url': video_urls[i],
-                        'format_id': self.get_format_name(fm),
-                        'ext': self.parse_ext_l(fm),
-                        'filesize': int(data1['segs'][fm][i]['size'])
-                    }
-                )
+                entries[i]['formats'].append({
+                    'url': video_urls[i],
+                    'format_id': self.get_format_name(fm),
+                    'ext': self.parse_ext_l(fm),
+                    'filesize': int(data1['segs'][fm][i]['size'])
+                })
 
         for i in range(len(entries)):
-            entries[i].update(
-                {
-                    'id': '%s_part%d' % (video_id, i + 1),
-                    'title': title,
-                }
-            )
+            entries[i].update({
+                'id': '%s_part%d' % (video_id, i + 1),
+                'title': title,
+            })
 
         return {
             '_type': 'multi_video',

From 0501bfa159db5b5e8ed7fd1ed966b9989becb3e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:15:30 +0800
Subject: [PATCH 0422/2145] [YoutubeDL] Youku extractor now uses the standard
 format selection

---
 youtube_dl/YoutubeDL.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aacec2958..6e4b6f566 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1033,12 +1033,6 @@ class YoutubeDL(object):
             info_dict['id'], info_dict.get('subtitles'),
             info_dict.get('automatic_captions'))
 
-        # This extractors handle format selection themselves
-        if info_dict['extractor'] in ['Youku']:
-            if download:
-                self.process_info(info_dict)
-            return info_dict
-
         # We now pick which formats have to be downloaded
         if info_dict.get('formats') is None:
             # There's only one format available

From 7f0172b3e5e0da2a19708fdf3ec1b521a6e2656f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 22:29:41 +0600
Subject: [PATCH 0423/2145] Credit @jackyzy823 for iqiyi

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bf2a25cb8..889d599a2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -127,3 +127,4 @@ Julian Richen
 Ping O.
 Mister Hat
 Peter Ding
+jackyzy823

From f3aecb27a4d7b178ae66b4a294cff5dbe9bb2b18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Jun 2015 14:41:52 +0200
Subject: [PATCH 0424/2145] [youku] Simplify a bit the 'entries' construction

Mainly avoid having to use an index.
---
 youtube_dl/extractor/youku.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index cab5be3a4..ced3a10cd 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -210,25 +210,23 @@ class YoukuIE(InfoExtractor):
         video_urls_dict = self.construct_video_urls(data1, data2)
 
         # construct info
-        entries = []
+        entries = [{
+            'id': '%s_part%d' % (video_id, i + 1),
+            'title': title,
+            'formats': [],
+            # some formats are not available for all parts, we have to detect
+            # which one has all
+        } for i in range(max(len(v) for v in data1['segs'].values()))]
         for fm in data1['streamtypes']:
             video_urls = video_urls_dict[fm]
-            for i in range(len(video_urls)):
-                if len(entries) < i + 1:
-                    entries.append({'formats': []})
-                entries[i]['formats'].append({
-                    'url': video_urls[i],
+            for video_url, seg, entry in zip(video_urls, data1['segs'][fm], entries):
+                entry['formats'].append({
+                    'url': video_url,
                     'format_id': self.get_format_name(fm),
                     'ext': self.parse_ext_l(fm),
-                    'filesize': int(data1['segs'][fm][i]['size'])
+                    'filesize': int(seg['size']),
                 })
 
-        for i in range(len(entries)):
-            entries[i].update({
-                'id': '%s_part%d' % (video_id, i + 1),
-                'title': title,
-            })
-
         return {
             '_type': 'multi_video',
             'id': video_id,

From 447053668fbed993f6f4fd2e06d9282ea30224bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Jun 2015 21:19:18 +0600
Subject: [PATCH 0425/2145] [spankwire] Fix extraction

---
 youtube_dl/extractor/spankwire.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 06d6e6640..bff75d6b2 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -27,7 +27,7 @@ class SpankwireIE(InfoExtractor):
             'description': 'Crazy Bitch X rated music video.',
             'uploader': 'oreusz',
             'uploader_id': '124697',
-            'upload_date': '20070508',
+            'upload_date': '20070507',
             'age_limit': 18,
         }
     }
@@ -44,7 +44,7 @@ class SpankwireIE(InfoExtractor):
         title = self._html_search_regex(
             r'<h1>([^<]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'<div\s+id="descriptionContent">([^<]+)<',
+            r'(?s)<div\s+id="descriptionContent">(.+?)</div>',
             webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
             r'playerData\.screenShot\s*=\s*["\']([^"\']+)["\']',
@@ -64,12 +64,12 @@ class SpankwireIE(InfoExtractor):
             r'<div id="viewsCounter"><span>([\d,\.]+)</span> views</div>',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'Comments<span[^>]+>\s*\(([\d,\.]+)\)</span>',
+            r'<span\s+id="spCommentCount"[^>]*>([\d,\.]+)</span>',
             webpage, 'comment count', fatal=False))
 
         video_urls = list(map(
             compat_urllib_parse.unquote,
-            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
+            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',

From 14835de9fb41798c8e6e731a3f07ae871770666f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 16 Jun 2015 18:10:31 -0300
Subject: [PATCH 0426/2145] Use shlex.split for --pp-params and update related
 docs.

---
 README.md                          | 2 +-
 youtube_dl/YoutubeDL.py            | 1 +
 youtube_dl/__init__.py             | 6 ++++--
 youtube_dl/options.py              | 4 ++--
 youtube_dl/postprocessor/common.py | 3 ++-
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 726ec9cf2..813ac4a15 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --pp-params                      Extra parameters for video post-processor. The params will be splited on spaces.
+    --pp-params                      Extra parameters for video post-processor.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..3bfe30c76 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -261,6 +261,7 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
+    pp_params:         Extra parameters for external apps, like avconv.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5b28e4817..8b54d4ae2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -171,8 +171,10 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
-    if opts.pp_params is not None:
-        opts.pp_params = opts.pp_params.split()
+    if opts.pp_params is None:
+        opts.pp_params = []
+    else:
+        opts.pp_params = shlex.split(opts.pp_params)
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ceb4b5f38..fbba9b9d8 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -689,8 +689,8 @@ def parseOpts(overrideArguments=None):
         help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
     postproc.add_option(
         '--pp-params',
-        dest='pp_params', default=None,
-        help='Extra parameters for video post-processor. The params will be splited on spaces.')
+        dest='pp_params', default=None, metavar='ARGS',
+        help='Extra parameters for video post-processor.')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 3b0e8ddd8..d944d9367 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -22,7 +22,8 @@ class PostProcessor(object):
     of the chain is reached.
 
     PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects.
+    to InfoExtractor objects. And it can receive parameters from CLI trough
+    --pp-params.
     """
 
     _downloader = None

From 028a33d7f2a0bc028f533530d2722b57b31dabdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:27:38 +0600
Subject: [PATCH 0427/2145] [lifenews] Fix extraction

---
 youtube_dl/extractor/lifenews.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 42cb6e35f..373122c93 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -82,10 +82,11 @@ class LifeNewsIE(InfoExtractor):
         view_count = self._html_search_regex(
             r'<div class=\'views\'>\s*(\d+)\s*</div>', webpage, 'view count', fatal=False)
         comment_count = self._html_search_regex(
-            r'<div class=\'comments\'>\s*<span class=\'counter\'>\s*(\d+)\s*</span>', webpage, 'comment count', fatal=False)
+            r'=\'commentCount\'[^>]*>\s*(\d+)\s*<',
+            webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
-            r'<time datetime=\'([^\']+)\'>', webpage, 'upload date', fatal=False)
+            r'<time[^>]*datetime=\'([^\']+)\'', webpage, 'upload date', fatal=False)
         if upload_date is not None:
             upload_date = unified_strdate(upload_date)
 

From 062a3fdf36dee6515c9e3b103c39ac2691196e95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:28:47 +0600
Subject: [PATCH 0428/2145] [lifenews] Modernize

---
 youtube_dl/extractor/lifenews.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 373122c93..1f07db3ec 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     int_or_none,
+    remove_end,
     unified_strdate,
     ExtractorError,
 )
@@ -72,10 +73,9 @@ class LifeNewsIE(InfoExtractor):
         if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 
-        title = self._og_search_title(webpage)
-        TITLE_SUFFIX = ' - Первый по срочным новостям — LIFE | NEWS'
-        if title.endswith(TITLE_SUFFIX):
-            title = title[:-len(TITLE_SUFFIX)]
+        title = remove_end(
+            self._og_search_title(webpage),
+            ' - Первый по срочным новостям — LIFE | NEWS')
 
         description = self._og_search_description(webpage)
 

From a69e8bfdd9e94537eb72a358c1b62dbea8a59a04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:29:15 +0600
Subject: [PATCH 0429/2145] [lifenews] Fix tests

---
 youtube_dl/extractor/lifenews.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 1f07db3ec..f8cbca7b3 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -40,7 +40,6 @@ class LifeNewsIE(InfoExtractor):
             'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
             'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
             'upload_date': '20150402',
-            'uploader': 'embed.life.ru',
         }
     }, {
         'url': 'http://lifenews.ru/news/153461',
@@ -51,7 +50,6 @@ class LifeNewsIE(InfoExtractor):
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
             'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
             'upload_date': '20150505',
-            'uploader': 'embed.life.ru',
         }
     }, {
         'url': 'http://lifenews.ru/video/13035',

From 0d0d5d37174ed611dd823c5be025c49e73d83d1d Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 13:59:37 +0800
Subject: [PATCH 0430/2145] [qqmusic] Add support for playlists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..d03577cdf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -427,6 +427,7 @@ from .qqmusic import (
     QQMusicSingerIE,
     QQMusicAlbumIE,
     QQMusicToplistIE,
+    QQMusicPlaylistIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..f9aafcd28 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    clean_html,
 )
 from ..compat import compat_urllib_request
 
@@ -243,3 +244,36 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_name = topinfo.get('ListName')
         list_description = topinfo.get('info')
         return self.playlist_result(entries, list_id, list_name, list_description)
+
+
+class QQMusicPlaylistIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:playlist'
+    _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://y.qq.com/#type=taoge&id=3462654915',
+        'info_dict': {
+            'id': '3462654915',
+            'title': '韩国5月新歌精选下旬',
+            'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
+        },
+        'playlist_count': 40,
+    }
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        list_json = self._download_json(
+            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
+            % list_id, list_id, 'Download list page',
+            transform_source=strip_jsonp)['cdlist'][0]
+
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+            ) for song in list_json['songlist']
+        ]
+
+        list_name = list_json['dissname']
+        list_description = clean_html(unescapeHTML(list_json.get('desc')))
+        return self.playlist_result(entries, list_id, list_name, list_description)

From 8f73e89ca0ecde0a8bbd1f1463e9a06a53c6b573 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 14:15:54 +0800
Subject: [PATCH 0431/2145] [kuwo] New extractor for kuwo.cn

---
 youtube_dl/extractor/__init__.py |   9 +
 youtube_dl/extractor/kuwo.py     | 326 +++++++++++++++++++++++++++++++
 2 files changed, 335 insertions(+)
 create mode 100644 youtube_dl/extractor/kuwo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..82cd85c44 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -257,6 +257,15 @@ from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
+from .kuwo import (
+    KuwoIE,
+    KuwoAlbumIE,
+    KuwoChartIE,
+    KuwoSingerIE,
+    KuwoSingerMusicIE,
+    KuwoCategoryIE,
+    KuwoMvIE,
+)
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .letv import (
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
new file mode 100644
index 000000000..6a96a1aa4
--- /dev/null
+++ b/youtube_dl/extractor/kuwo.py
@@ -0,0 +1,326 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    get_element_by_id,
+    clean_html,
+    ExtractorError,
+)
+
+
+class KuwoIE(InfoExtractor):
+    IE_NAME = 'kuwo:song'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/yinyue/635632/',
+        'info_dict': {
+            'id': '635632',
+            'ext': 'ape',
+            'title': '爱我别走',
+            'creator': '张震岳',
+            'upload_date': '20080122',
+            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
+        },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/6446136/',
+        'info_dict': {
+            'id': '6446136',
+            'ext': 'mp3',
+            'title': '心',
+            'creator': 'IU',
+            'upload_date': '20150518',
+        },
+        'params': {
+            'format': 'mp3-320'
+        },
+    }]
+    _FORMATS = [
+        {'format': 'ape', 'ext': 'ape', 'preference': 100},
+        {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
+        {'format': 'mp3-192', 'ext': 'mp3', 'br': '192kmp3', 'abr': 192, 'preference': 70},
+        {'format': 'mp3-128', 'ext': 'mp3', 'br': '128kmp3', 'abr': 128, 'preference': 60},
+        {'format': 'wma', 'ext': 'wma', 'preference': 20},
+        {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
+    ]
+
+    def _get_formats(self, song_id):
+        formats = []
+        for file_format in self._FORMATS:
+            song_url = self._download_webpage(
+                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
+                (file_format['ext'], file_format.get('br', ''), song_id),
+                song_id, note="Download %s url info" % file_format["format"],
+            )
+            if song_url.startswith('http://') or song_url.startswith('https://'):
+                formats.append({
+                    'url': song_url,
+                    'format_id': file_format['format'],
+                    'format': file_format['format'],
+                    'preference': file_format['preference'],
+                    'abr': file_format.get('abr'),
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, song_id, note='Download song detail info',
+            errnote='Unable to get song detail info')
+
+        song_name = self._html_search_regex(
+            r'<h1 title="(.+?)">', webpage, 'song name')
+        singer_name = self._html_search_regex(
+            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
+            flags=re.DOTALL, default=None)
+        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
+        if lrc_content == '暂无':     # indicates no lyrics
+            lrc_content = None
+
+        formats = self._get_formats(song_id)
+
+        album_id = self._html_search_regex(
+            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
+            webpage, 'album id', default=None, fatal=False)
+
+        publish_time = None
+        if album_id is not None:
+            album_info_page = self._download_webpage(
+                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
+                note='Download album detail info',
+                errnote='Unable to get album detail info')
+
+            publish_time = self._html_search_regex(
+                r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
+                'publish time', default=None)
+            if publish_time:
+                publish_time = publish_time.replace('-', '')
+
+        return {
+            'id': song_id,
+            'title': song_name,
+            'creator': singer_name,
+            'upload_date': publish_time,
+            'description': lrc_content,
+            'formats': formats,
+        }
+
+
+class KuwoAlbumIE(InfoExtractor):
+    IE_NAME = 'kuwo:album'
+    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/album/502294/',
+        'info_dict': {
+            'id': '502294',
+            'title': 'M',
+            'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
+        },
+        'playlist_count': 2,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, album_id, note='Download album info',
+            errnote='Unable to get album info')
+
+        album_name = self._html_search_regex(
+            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
+            'album name', flags=re.DOTALL)
+        album_intro = clean_html(
+            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
+                webpage)
+        ]
+        return self.playlist_result(entries, album_id, album_name, album_intro)
+
+
+class KuwoChartIE(InfoExtractor):
+    IE_NAME = 'kuwo:chart'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
+    _TEST = {
+        'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
+        'info_dict': {
+            'id': '香港中文龙虎榜',
+            'title': '香港中文龙虎榜',
+            'description': 're:[0-9]{4}第[0-9]{2}期',
+        },
+        'playlist_mincount': 10,
+    }
+
+    def _real_extract(self, url):
+        chart_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, chart_id, note='Download chart info',
+            errnote='Unable to get chart info')
+
+        chart_name = self._html_search_regex(
+            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
+
+        chart_desc = self._html_search_regex(
+            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
+        ]
+        return self.playlist_result(entries, chart_id, chart_name, chart_desc)
+
+
+class KuwoSingerIE(InfoExtractor):
+    IE_NAME = 'kuwo:singer'
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/$'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
+        'info_dict': {
+            'id': 'bruno+mars',
+            'title': 'Bruno Mars',
+        },
+        'playlist_count': 10,
+    }
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, singer_id, note='Download singer info',
+            errnote='Unable to get singer info')
+
+        singer_name = self._html_search_regex(
+            r'姓名：<span>(.+?)</span>', webpage, 'singer name')
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>',
+                webpage, flags=re.DOTALL)
+        ]
+        return self.playlist_result(entries, singer_id, singer_name)
+
+
+class KuwoSingerMusicIE(InfoExtractor):
+    IE_NAME = 'kuwo:singermusic'
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/music(_[0-9]+)?.htm'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
+        'info_dict': {
+            'id': 'Ali',
+            'title': 'Ali的热门歌曲',
+        },
+        'playlist_mincount': 95,
+    }
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+
+        list_name = None
+        entries = []
+        for page_num in itertools.count(1):
+            webpage = self._download_webpage(
+                'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
+                singer_id, note='Download song list page #%d' % page_num,
+                errnote='Unable to get song list page #%d' % page_num)
+
+            if list_name is None:
+                list_name = self._html_search_regex(
+                    r'<h1>([^<>]+)<span>', webpage, 'list name')
+
+            entries.extend([
+                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+                for song_id in re.findall(
+                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                    webpage)
+            ])
+            if not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+                break
+
+        return self.playlist_result(entries, singer_id, list_name)
+
+
+class KuwoCategoryIE(InfoExtractor):
+    IE_NAME = 'kuwo:category'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
+    _TEST = {
+        'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
+        'info_dict': {
+            'id': '86375',
+            'title': '八十年代精选',
+            'description': '这些都是属于八十年代的回忆！',
+        },
+        'playlist_count': 30,
+    }
+
+    def _real_extract(self, url):
+        category_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, category_id, note='Download category info',
+            errnote='Unable to get category info')
+
+        category_name = self._html_search_regex(
+            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
+        
+        category_desc = re.sub(
+            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
+        
+        jsonm = self._parse_json(self._html_search_regex(
+            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
+
+        entries = [
+            self.url_result(
+                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
+                'Kuwo', song['musicrid'])
+            for song in jsonm['musiclist']
+        ]
+        return self.playlist_result(entries, category_id, category_name, category_desc)
+
+
+class KuwoMvIE(KuwoIE):
+    IE_NAME = 'kuwo:mv'
+    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/mv/6480076/',
+        'info_dict': {
+            'id': '6480076',
+            'ext': 'mkv',
+            'title': '我们家MV',
+            'creator': '2PM',
+        },
+    }]
+    _FORMATS = KuwoIE._FORMATS + [
+        {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
+        {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
+    ]
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, song_id, note='Download mv detail info: %s' % song_id,
+            errnote='Unable to get mv detail info: %s' % song_id)
+
+        mobj = re.search(
+            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
+            webpage)
+        if mobj:
+            song_name = mobj.group('song')
+            singer_name = mobj.group('singer')
+        else:
+            raise ExtractorError("Unable to find song or singer names")
+
+        formats = self._get_formats(song_id)
+
+        return {
+            'id': song_id,
+            'title': song_name,
+            'creator': singer_name,
+            'formats': formats,
+        }

From 29b809de68aeefb5e991c75929ed3d03fb40c1f1 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 15:52:04 +0800
Subject: [PATCH 0432/2145] [qqmusic] Fix album extraction

---
 youtube_dl/extractor/qqmusic.py | 37 ++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..d9a783f8a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -163,31 +163,38 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
-    _TEST = {
-        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1&play=0',
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
-            'description': 'md5:d216c55a2d4b3537fe4415b8767d74d6',
+            'description': 'md5:712f0cdbfc7e776820d08150e6df593d',
         },
         'playlist_count': 4,
-    }
+    }, {
+        'url': 'http://y.qq.com/#type=album&mid=002Y5a3b3AlCu3',
+        'info_dict': {
+            'id': '002Y5a3b3AlCu3',
+            'title': '그리고...',
+            'description': 'md5:b1d133b8c9bac8fed4e1a97df759f4cf',
+        },
+        'playlist_count': 8,
+    }]
 
     def _real_extract(self, url):
         mid = self._match_id(url)
 
-        album_page = self._download_webpage(
-            self.qq_static_url('album', mid), mid, 'Download album page')
+        album = self._download_json(
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_album_info_cp.fcg?albummid=%s&format=json' % mid,
+            mid, 'Download album page')['data']
 
-        entries = self.get_entries_from_page(album_page)
-
-        album_name = self._html_search_regex(
-            r"albumname\s*:\s*'([^']+)',", album_page, 'album name',
-            default=None)
-
-        album_detail = self._html_search_regex(
-            r'<div class="album_detail close_detail">\s*<p>((?:[^<>]+(?:<br />)?)+)</p>',
-            album_page, 'album details', default=None)
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+            ) for song in album['list']
+        ]
+        album_name = album['name']
+        album_detail = album.get('desc')
 
         return self.playlist_result(entries, mid, album_name, album_detail)
 

From 5e3915cbe3fabd2dbb633131b851b1158c0bba7b Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 21:06:25 +0800
Subject: [PATCH 0433/2145] [qqmusic] Fix song extraction when certain formats
 are unavailable

---
 youtube_dl/extractor/qqmusic.py | 55 +++++++++++++++++++++++++++------
 1 file changed, 46 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..7ddc4ca25 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,8 +9,13 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    HEADRequest,
+    ExtractorError,
+)
+from ..compat import (
+    compat_urllib_request,
+    compat_HTTPError,
 )
-from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -26,6 +31,20 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
+            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/7/p/001IV22P1RDX7p.jpg',
+        }
+    }, {
+        'note': 'There is no mp3-320 version of this song.',
+        'url': 'http://y.qq.com/#type=song&mid=004MsGEo3DdNxV',
+        'md5': 'fa3926f0c585cda0af8fa4f796482e3e',
+        'info_dict': {
+            'id': '004MsGEo3DdNxV',
+            'ext': 'mp3',
+            'title': '如果',
+            'upload_date': '20050626',
+            'creator': '李季美',
+            'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
+            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/r/Q/0042owYj46IxrQ.jpg',
         }
     }]
 
@@ -68,6 +87,13 @@ class QQMusicIE(InfoExtractor):
         if lrc_content:
             lrc_content = lrc_content.replace('\\n', '\n')
 
+        thumbnail_url = None
+        albummid = self._search_regex(
+            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'], detail_info_page, 'album mid', default=None)
+        if albummid:
+            thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
+                            % (albummid[-2:-1], albummid[-1], albummid)
+
         guid = self.m_r_get_ruin()
 
         vkey = self._download_json(
@@ -77,14 +103,24 @@ class QQMusicIE(InfoExtractor):
 
         formats = []
         for format_id, details in self._FORMATS.items():
-            formats.append({
-                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (details['prefix'], mid, details['ext'], vkey, guid),
-                'format': format_id,
-                'format_id': format_id,
-                'preference': details['preference'],
-                'abr': details.get('abr'),
-            })
+            video_url = 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0' \
+                        % (details['prefix'], mid, details['ext'], vkey, guid)
+            req = HEADRequest(video_url)
+            try:
+                res = self._request_webpage(
+                    req, mid, note='Testing %s video URL' % format_id, fatal=False)
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [400, 404]:
+                    self.report_warning('Invalid %s video URL' % format_id, mid)
+            else:
+                if res:
+                    formats.append({
+                        'url': video_url,
+                        'format': format_id,
+                        'format_id': format_id,
+                        'preference': details['preference'],
+                        'abr': details.get('abr'),
+                    })
         self._sort_formats(formats)
 
         return {
@@ -94,6 +130,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': publish_time,
             'creator': singer,
             'description': lrc_content,
+            'thumbnail': thumbnail_url,
         }
 
 

From 0392ac98d2c5c5a6fd2ab51c51096f82312a287c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 21:13:03 +0800
Subject: [PATCH 0434/2145] [qqmusic] Fix code formatting

---
 youtube_dl/extractor/qqmusic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 7ddc4ca25..5a18191bc 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -89,7 +89,8 @@ class QQMusicIE(InfoExtractor):
 
         thumbnail_url = None
         albummid = self._search_regex(
-            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'], detail_info_page, 'album mid', default=None)
+            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'],
+            detail_info_page, 'album mid', default=None)
         if albummid:
             thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
                             % (albummid[-2:-1], albummid[-1], albummid)

From 974a6146fe71029a8dedf670fd72501da5ec3809 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 21:40:45 +0800
Subject: [PATCH 0435/2145] [bbccouk] Fix error handling

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 249bc6bbd..4b25b8c38 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -267,7 +267,7 @@ class BBCCoUkIE(InfoExtractor):
                 programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
 

From 25fa8d66e653d443522bf10140106f88711d1d66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 22:00:13 +0800
Subject: [PATCH 0436/2145] [bbccouk] Fix description (closes #6006)

---
 youtube_dl/extractor/bbccouk.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 4b25b8c38..0305f88b5 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -129,6 +129,20 @@ class BBCCoUkIE(InfoExtractor):
                 'skip_download': True,
             },
             'skip': 'geolocation',
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
+            'info_dict': {
+                'id': 'b05zmgw1',
+                'ext': 'flv',
+                'description': 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.',
+                'title': 'Royal Academy Summer Exhibition',
+                'duration': 3540,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'geolocation',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,
@@ -362,7 +376,7 @@ class BBCCoUkIE(InfoExtractor):
             formats, subtitles = self._download_media_selector(programme_id)
             title = self._og_search_title(webpage)
             description = self._search_regex(
-                r'<p class="medium-description">([^<]+)</p>',
+                r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', fatal=False)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)

From c33a8639a7c1f783297f29282b48fceec9a0b84a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 22:05:02 +0800
Subject: [PATCH 0437/2145] [postprocessor/embedthumbnail] Skipping if
 thumbnail file missing

Fixes #5996
---
 youtube_dl/postprocessor/embedthumbnail.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 774494efd..e19dbf73d 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -35,6 +35,11 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
 
         thumbnail_filename = info['thumbnails'][-1]['filename']
 
+        if not os.path.exists(encodeFilename(thumbnail_filename)):
+            self._downloader.report_warning(
+                'Skipping embedding the thumbnail because the file is missing.')
+            return [], info
+
         if info['ext'] == 'mp3':
             options = [
                 '-c', 'copy', '-map', '0', '-map', '1',

From 4d58b24c15ea0efc699a7ad7ee468245029da4e3 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 23:09:04 +0800
Subject: [PATCH 0438/2145] [qqmusic] Use _check_formats instead

---
 youtube_dl/extractor/qqmusic.py | 34 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 5a18191bc..7183c2bb1 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,13 +9,8 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
-    HEADRequest,
-    ExtractorError,
-)
-from ..compat import (
-    compat_urllib_request,
-    compat_HTTPError,
 )
+from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -104,24 +99,15 @@ class QQMusicIE(InfoExtractor):
 
         formats = []
         for format_id, details in self._FORMATS.items():
-            video_url = 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0' \
-                        % (details['prefix'], mid, details['ext'], vkey, guid)
-            req = HEADRequest(video_url)
-            try:
-                res = self._request_webpage(
-                    req, mid, note='Testing %s video URL' % format_id, fatal=False)
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [400, 404]:
-                    self.report_warning('Invalid %s video URL' % format_id, mid)
-            else:
-                if res:
-                    formats.append({
-                        'url': video_url,
-                        'format': format_id,
-                        'format_id': format_id,
-                        'preference': details['preference'],
-                        'abr': details.get('abr'),
-                    })
+            formats.append({
+                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
+                       % (details['prefix'], mid, details['ext'], vkey, guid),
+                'format': format_id,
+                'format_id': format_id,
+                'preference': details['preference'],
+                'abr': details.get('abr'),
+            })
+        self._check_formats(formats, mid)
         self._sort_formats(formats)
 
         return {

From accf79b1079302a339a3565e98ecfea5b33b5528 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Jun 2015 21:17:26 +0600
Subject: [PATCH 0439/2145] [viki] Add support for authentication (Closes
 #6005)

---
 youtube_dl/extractor/viki.py | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 7f2fb1ca8..52d10d242 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,5 +1,7 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import time
 import hmac
 import hashlib
@@ -11,6 +13,7 @@ from ..utils import (
     parse_age_limit,
     parse_iso8601,
 )
+from ..compat import compat_urllib_request
 from .common import InfoExtractor
 
 
@@ -23,7 +26,9 @@ class VikiBaseIE(InfoExtractor):
     _APP_VERSION = '2.2.5.1428709186'
     _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
 
-    def _prepare_call(self, path, timestamp=None):
+    _NETRC_MACHINE = 'viki'
+
+    def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
             timestamp = int(time.time())
@@ -33,17 +38,19 @@ class VikiBaseIE(InfoExtractor):
             query.encode('ascii'),
             hashlib.sha1
         ).hexdigest()
-        return self._API_URL_TEMPLATE % (query, sig)
+        url = self._API_URL_TEMPLATE % (query, sig)
+        return compat_urllib_request.Request(
+            url, json.dumps(post_data).encode('utf-8')) if post_data else url
 
-    def _call_api(self, path, video_id, note, timestamp=None):
+    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
         resp = self._download_json(
-            self._prepare_call(path, timestamp), video_id, note)
+            self._prepare_call(path, timestamp, post_data), video_id, note)
 
         error = resp.get('error')
         if error:
             if error == 'invalid timestamp':
                 resp = self._download_json(
-                    self._prepare_call(path, int(resp['current_timestamp'])),
+                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
                     video_id, '%s (retry)' % note)
                 error = resp.get('error')
             if error:
@@ -56,6 +63,23 @@ class VikiBaseIE(InfoExtractor):
             '%s returned error: %s' % (self.IE_NAME, error),
             expected=True)
 
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'login_id': username,
+            'password': password,
+        }
+
+        self._call_api(
+            'sessions.json', None,
+            'Logging in as %s' % username, post_data=login_form)
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'

From 360075e28a4c009babb62e0ed8141ed5244cb3f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Jun 2015 22:26:17 +0600
Subject: [PATCH 0440/2145] [pornhub] Extend _VALID_URL (Closes #6019)

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 8565d7551..8172bc997 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -19,8 +19,8 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-f]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
+    _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
         'info_dict': {
@@ -30,7 +30,10 @@ class PornHubIE(InfoExtractor):
             "title": "Seductive Indian beauty strips down and fingers her pink pussy",
             "age_limit": 18
         }
-    }
+    }, {
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_url(cls, webpage):

From 9e96dc8b3561c1e6e62ce6a34efba485e5e49054 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 01:36:59 -0500
Subject: [PATCH 0441/2145] Support BBC News (bbc.com/news)

---
 docs/supportedsites.md           |   1 +
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/bbcnews.py  | 162 +++++++++++++++++++++++++++++++
 3 files changed, 164 insertions(+)
 create mode 100644 youtube_dl/extractor/bbcnews.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 220e52b98..d4ccbbd3a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,6 +50,7 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
+ - **bbc.com**: BBC news videos
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..51d2d20e9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -36,6 +36,7 @@ from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
+from .bbcnews import BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
new file mode 100644
index 000000000..b10e30a81
--- /dev/null
+++ b/youtube_dl/extractor/bbcnews.py
@@ -0,0 +1,162 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+from ..compat import compat_HTTPError
+import re
+from .bbccouk import BBCCoUkIE
+
+class BBCNewsIE(BBCCoUkIE):
+    IE_NAME = 'bbc.com'
+    IE_DESC = 'BBC news'
+    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+        },
+        'playlist_count': 2,
+    },{
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+        },
+        'playlist_count': 9,
+    },{
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'mp4',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _duration_str2int(self, str):
+        if not str:
+            return None
+        ret = re.match(r'^\d+$', str)
+        if ret:
+            return int(ret.group(0))
+        ret = re.match(r'PT((?P<h>\d+)H)?((?P<m>\d+)M)?(?P<s>\d+)S$', str)
+        if ret:
+            total=int(ret.group('s'))
+            if ret.group('m'):
+                total+=(int(ret.group('m'))*60)
+            if ret.group('h'):
+                total+=(int(ret.group('h'))*3600)
+            return total
+        return None
+
+    def _download_media_selector(self, programme_id):
+        # bbc news uses http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ not
+        # http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/
+        # Could add third urlspec arg to BBCCoUkIE._download_media_selector instead of duplicating it
+
+        try:
+            media_selection = self._download_xml(
+               'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' % programme_id,
+                programme_id, 'Downloading media selection XML')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+            else:
+                raise
+        formats = []
+        subtitles = None
+
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self.extract_subtitles(media, programme_id)
+
+        formats = []
+        subtitles = None
+
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self.extract_subtitles(media, programme_id)
+
+        return formats, subtitles
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+
+        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
+        if pubdate:
+           pubdate = pubdate.replace('-','')
+
+        ret = []
+        # works with bbc.com/news/something-something-123456 articles
+        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
+        if not matches:
+           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+           # in http://www.bbc.com/news/video_and_audio/international
+           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+        if not matches:
+           raise ExtractorError('No video found', expected=True)
+
+        for ent in matches:
+            jent = self._parse_json(ent,list_id)
+
+            programme_id = jent.get('externalId',None)
+            xml_url = jent.get('href', None)
+
+            title = jent['caption']
+            duration = self._duration_str2int(jent.get('duration',None))
+            description = list_title + ' - ' + jent.get('caption','')
+            thumbnail = None
+            if jent.has_key('image'):
+               thumbnail=jent['image'].get('href',None)
+
+            if programme_id:
+               formats, subtitles = self._download_media_selector(programme_id)
+            elif xml_url:
+               # Cheap fallback
+               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+               formats, subtitles = self._download_media_selector(programme_id)
+            else:
+               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+               
+            self._sort_formats(formats)
+
+            ret.append( {
+                'id': programme_id,
+                'uploader': 'BBC News',
+                'upload_date': pubdate,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            } )
+
+        if len(ret) > 0:
+           return self.playlist_result(ret, list_id, list_title)
+        raise ExtractorError('No video found', expected=True)

From a8b081a0523c412fd4e01d5cddec7ae382c4793e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 01:52:25 -0500
Subject: [PATCH 0442/2145] BBCNewsIE: eliminate redundant function.
 BBCCoUkIE._download_media_selector: use class variable instead of hardcoded
 string for mediaselector_url template.

---
 youtube_dl/extractor/bbccouk.py |  4 +++-
 youtube_dl/extractor/bbcnews.py | 42 ++-------------------------------
 2 files changed, 5 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 0305f88b5..dcc5fc2fa 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -15,6 +15,8 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -277,7 +279,7 @@ class BBCCoUkIE(InfoExtractor):
     def _download_media_selector(self, programme_id):
         try:
             media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
+                self.mediaselector_url % programme_id,
                 programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
index b10e30a81..9bb8d42e6 100644
--- a/youtube_dl/extractor/bbcnews.py
+++ b/youtube_dl/extractor/bbcnews.py
@@ -14,6 +14,8 @@ class BBCNewsIE(BBCCoUkIE):
     IE_DESC = 'BBC news'
     _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
 
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+
     _TESTS = [{
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
@@ -59,46 +61,6 @@ class BBCNewsIE(BBCCoUkIE):
             return total
         return None
 
-    def _download_media_selector(self, programme_id):
-        # bbc news uses http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ not
-        # http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/
-        # Could add third urlspec arg to BBCCoUkIE._download_media_selector instead of duplicating it
-
-        try:
-            media_selection = self._download_xml(
-               'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' % programme_id,
-                programme_id, 'Downloading media selection XML')
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
-            else:
-                raise
-        formats = []
-        subtitles = None
-
-        for media in self._extract_medias(media_selection):
-            kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
-            elif kind == 'captions':
-                subtitles = self.extract_subtitles(media, programme_id)
-
-        formats = []
-        subtitles = None
-
-        for media in self._extract_medias(media_selection):
-            kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
-            elif kind == 'captions':
-                subtitles = self.extract_subtitles(media, programme_id)
-
-        return formats, subtitles
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)

From d5552a3477a0970f4aaaa746ce07c816267bb9cf Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 06:25:50 -0500
Subject: [PATCH 0443/2145] bbcnews: Switch to parse_duration, revert change to
 docs/supportedsites.md

---
 docs/supportedsites.md          |  1 -
 youtube_dl/extractor/bbcnews.py | 19 ++-----------------
 2 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d4ccbbd3a..220e52b98 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,7 +50,6 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
- - **bbc.com**: BBC news videos
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
index 9bb8d42e6..fd4a5e38f 100644
--- a/youtube_dl/extractor/bbcnews.py
+++ b/youtube_dl/extractor/bbcnews.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    parse_duration,
     int_or_none,
 )
 from ..compat import compat_HTTPError
@@ -45,22 +46,6 @@ class BBCNewsIE(BBCCoUkIE):
         }
     }]
 
-    def _duration_str2int(self, str):
-        if not str:
-            return None
-        ret = re.match(r'^\d+$', str)
-        if ret:
-            return int(ret.group(0))
-        ret = re.match(r'PT((?P<h>\d+)H)?((?P<m>\d+)M)?(?P<s>\d+)S$', str)
-        if ret:
-            total=int(ret.group('s'))
-            if ret.group('m'):
-                total+=(int(ret.group('m'))*60)
-            if ret.group('h'):
-                total+=(int(ret.group('h'))*3600)
-            return total
-        return None
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
@@ -88,7 +73,7 @@ class BBCNewsIE(BBCCoUkIE):
             xml_url = jent.get('href', None)
 
             title = jent['caption']
-            duration = self._duration_str2int(jent.get('duration',None))
+            duration = parse_duration(jent.get('duration',None))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):

From c9bebed294dd29d9188265c8f7bfb0e1b43406ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 20:52:44 +0600
Subject: [PATCH 0444/2145] [youtube] Add itag 59 and 78 (Closes #5979)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9e2671192..a3da56c14 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -234,6 +234,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '44': {'ext': 'webm', 'width': 854, 'height': 480},
         '45': {'ext': 'webm', 'width': 1280, 'height': 720},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480},
 
 
         # 3d videos

From cbcd1a5474dd8b39e68b0d2bbc493701c655a2d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 21:57:31 +0600
Subject: [PATCH 0445/2145] [dramafever] Add support for authentication (Closes
 #6017)

---
 youtube_dl/extractor/dramafever.py | 38 ++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index a34aad486..cfbcddcef 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -6,6 +6,8 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
+    compat_urllib_parse,
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,7 +19,39 @@ from ..utils import (
 )
 
 
-class DramaFeverIE(InfoExtractor):
+class DramaFeverBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
+    _NETRC_MACHINE = 'dramafever'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'username': username,
+            'password': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if all(logout_pattern not in response
+               for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
+            error = self._html_search_regex(
+                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                response, 'error message', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+
+class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TEST = {
@@ -97,7 +131,7 @@ class DramaFeverIE(InfoExtractor):
         }
 
 
-class DramaFeverSeriesIE(InfoExtractor):
+class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{

From 10464af5d1d03a3461286a601ae7db91c5a8141c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 22:02:07 +0600
Subject: [PATCH 0446/2145] [dramafever:series] Fix extraction while
 authenticated

---
 youtube_dl/extractor/dramafever.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index cfbcddcef..ca41a3abf 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -185,8 +185,11 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
                 % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
                 series_id, 'Downloading episodes JSON page #%d' % page_num)
             for episode in episodes.get('value', []):
+                episode_url = episode.get('episode_url')
+                if not episode_url:
+                    continue
                 entries.append(self.url_result(
-                    compat_urlparse.urljoin(url, episode['episode_url']),
+                    compat_urlparse.urljoin(url, episode_url),
                     'DramaFever', episode.get('guid')))
             if page_num == episodes['num_pages']:
                 break

From 385c3e5e91680dcc6573f05e6b30fdf45048503e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Jun 2015 00:10:08 +0600
Subject: [PATCH 0447/2145] [pinkbike] Improve and simplify

---
 youtube_dl/extractor/pinkbike.py | 88 ++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 45c0b1377..745433b48 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -7,12 +7,14 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     remove_end,
-    remove_start
+    remove_start,
+    str_to_int,
+    unified_strdate,
 )
 
 
 class PinkbikeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pinkbike\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?pinkbike\.com/video/|es\.pinkbike\.org/i/kvid/kvid-y5\.swf\?id=)(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.pinkbike.com/video/402811/',
         'md5': '4814b8ca7651034cd87e3361d5c2155a',
@@ -20,67 +22,75 @@ class PinkbikeIE(InfoExtractor):
             'id': '402811',
             'ext': 'mp4',
             'title': 'Brandon Semenuk - RAW 100',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'location': 'Victoria, British Columbia, Canada',
-            'uploader_id': 'revelco',
-            'upload_date': '20150406',
             'description': 'Official release: www.redbull.ca/rupertwalker',
-            'duration': 100
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 100,
+            'upload_date': '20150406',
+            'uploader': 'revelco',
+            'location': 'Victoria, British Columbia, Canada',
+            'view_count': int,
+            'comment_count': int,
         }
     }, {
-        'url': 'http://www.pinkbike.com/video/406629/',
-        'md5': 'c7a3e19a2bd5cde5a1cda6b2b46caa74',
-        'info_dict': {
-            'id': '406629',
-            'ext': 'mp4',
-            'title': 'Chromag: Reece Wallace in Utah',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'location': 'Whistler, British Columbia, Canada',
-            'uploader_id': 'Chromagbikes',
-            'upload_date': '20150505',
-            'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
-            'duration': 180
-        }
+        'url': 'http://es.pinkbike.org/i/kvid/kvid-y5.swf?id=406629',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        title = remove_end(title, ' Video - Pinkbike')
+        webpage = self._download_webpage(
+            'http://www.pinkbike.com/video/%s' % video_id, video_id)
 
-        description = self._html_search_meta('description', webpage, 'description')
-        description = remove_start(description, title + '. ')
+        formats = []
+        for _, format_id, src in re.findall(
+            r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None))
+            formats.append({
+                'url': src,
+                'format_id': format_id,
+                'height': height,
+            })
+        self._sort_formats(formats)
 
+        title = remove_end(self._og_search_title(webpage), ' Video - Pinkbike')
+        description = self._html_search_regex(
+            r'(?s)id="media-description"[^>]*>(.+?)<',
+            webpage, 'description', default=None) or remove_start(
+            self._og_search_description(webpage), title + '. ')
+        thumbnail = self._og_search_thumbnail(webpage)
         duration = int_or_none(self._html_search_meta(
             'video:duration', webpage, 'duration'))
 
-        uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
-
-        upload_date = self._html_search_regex(
-            r'class="fullTime"\s*title="([0-9]{4}(?:-[0-9]{2}){2})"',
-            webpage, 'upload_date')
-        upload_date = upload_date.replace('-', '')
+        uploader = self._search_regex(
+            r'un:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
+        upload_date = unified_strdate(self._search_regex(
+            r'class="fullTime"[^>]+title="([^"]+)"',
+            webpage, 'upload date', fatal=False))
 
         location = self._html_search_regex(
-            r'<dt>Location</dt>\n?\s*<dd>\n?(.*?)\s*<img',
-            webpage, 'location')
+            r'(?s)<dt>Location</dt>\s*<dd>(.+?)<',
+            webpage, 'location', fatal=False)
 
-        formats = re.findall(
-            r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
-            webpage)
+        def extract_count(webpage, label):
+            return str_to_int(self._search_regex(
+                r'<span[^>]+class="stat-num"[^>]*>([\d,.]+)</span>\s*<span[^>]+class="stat-label"[^>]*>%s' % label,
+                webpage, label, fatal=False))
 
-        formats = [{'url': fmt[1], 'height': int_or_none(fmt[0])} for fmt in formats]
+        view_count = extract_count(webpage, 'Views')
+        comment_count = extract_count(webpage, 'Comments')
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
-            'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
-            'uploader_id': uploader_id,
             'upload_date': upload_date,
+            'uploader': uploader,
             'location': location,
+            'view_count': view_count,
+            'comment_count': comment_count,
             'formats': formats
         }

From 16d6973f8a9eb5a70c12d82aa40f57c2b4aa8c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Jun 2015 00:49:28 +0600
Subject: [PATCH 0448/2145] [viki] Pass session token around (#6005)

---
 youtube_dl/extractor/viki.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 52d10d242..51cdc6b65 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -28,11 +28,15 @@ class VikiBaseIE(InfoExtractor):
 
     _NETRC_MACHINE = 'viki'
 
+    _token = None
+
     def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
             timestamp = int(time.time())
         query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        if self._token:
+            query += '&token=%s' % self._token
         sig = hmac.new(
             self._APP_SECRET.encode('ascii'),
             query.encode('ascii'),
@@ -76,10 +80,14 @@ class VikiBaseIE(InfoExtractor):
             'password': password,
         }
 
-        self._call_api(
+        login = self._call_api(
             'sessions.json', None,
             'Logging in as %s' % username, post_data=login_form)
 
+        self._token = login.get('token')
+        if not self._token:
+            self.report_warning('Unable to get session token, login has probably failed')
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'

From 10273d6e0846cd8f3762e3777712d5cd2a0cafcd Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:22:13 -0500
Subject: [PATCH 0449/2145] toss new stuff into old file

---
 youtube_dl/extractor/__init__.py |   3 +-
 youtube_dl/extractor/bbccouk.py  | 101 ++++++++++++++++++++++++++++
 youtube_dl/extractor/bbcnews.py  | 109 -------------------------------
 3 files changed, 102 insertions(+), 111 deletions(-)
 delete mode 100644 youtube_dl/extractor/bbcnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 51d2d20e9..f9f7bdfaf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -35,8 +35,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE
-from .bbcnews import BBCNewsIE
+from .bbccouk import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index dcc5fc2fa..ea682fb6f 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -5,9 +5,11 @@ import xml.etree.ElementTree
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    parse_duration,
     int_or_none,
 )
 from ..compat import compat_HTTPError
+import re
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -394,3 +396,102 @@ class BBCCoUkIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class BBCNewsIE(BBCCoUkIE):
+    IE_NAME = 'bbc.com'
+    IE_DESC = 'BBC news'
+    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+
+    _TESTS = [{
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+        },
+        'playlist_count': 2,
+    },{
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+        },
+        'playlist_count': 9,
+    },{
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'mp4',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+
+        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
+        if pubdate:
+           pubdate = pubdate.replace('-','')
+
+        ret = []
+        # works with bbc.com/news/something-something-123456 articles
+        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
+        if not matches:
+           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+           # in http://www.bbc.com/news/video_and_audio/international
+           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+        if not matches:
+           raise ExtractorError('No video found', expected=True)
+
+        for ent in matches:
+            jent = self._parse_json(ent,list_id)
+
+            programme_id = jent.get('externalId',None)
+            xml_url = jent.get('href', None)
+
+            title = jent['caption']
+            duration = parse_duration(jent.get('duration',None))
+            description = list_title + ' - ' + jent.get('caption','')
+            thumbnail = None
+            if jent.has_key('image'):
+               thumbnail=jent['image'].get('href',None)
+
+            if programme_id:
+               formats, subtitles = self._download_media_selector(programme_id)
+            elif xml_url:
+               # Cheap fallback
+               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+               formats, subtitles = self._download_media_selector(programme_id)
+            else:
+               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+               
+            self._sort_formats(formats)
+
+            ret.append( {
+                'id': programme_id,
+                'uploader': 'BBC News',
+                'upload_date': pubdate,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            } )
+
+        if len(ret) > 0:
+           return self.playlist_result(ret, list_id, list_title)
+        raise ExtractorError('No video found', expected=True)
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
deleted file mode 100644
index fd4a5e38f..000000000
--- a/youtube_dl/extractor/bbcnews.py
+++ /dev/null
@@ -1,109 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    parse_duration,
-    int_or_none,
-)
-from ..compat import compat_HTTPError
-import re
-from .bbccouk import BBCCoUkIE
-
-class BBCNewsIE(BBCCoUkIE):
-    IE_NAME = 'bbc.com'
-    IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
-
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-
-    _TESTS = [{
-        'url': 'http://www.bbc.com/news/world-europe-32668511',
-        'info_dict': {
-            'id': 'world-europe-32668511',
-            'title': 'Russia stages massive WW2 parade despite Western boycott',
-        },
-        'playlist_count': 2,
-    },{
-        'url': 'http://www.bbc.com/news/business-28299555',
-        'info_dict': {
-            'id': 'business-28299555',
-            'title': 'Farnborough Airshow: Video highlights',
-        },
-        'playlist_count': 9,
-    },{
-        'url': 'http://www.bbc.com/news/world-europe-32041533',
-        'note': 'Video',
-        'info_dict': {
-            'id': 'p02mprgb',
-            'ext': 'mp4',
-            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'duration': 47,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }]
-
-    def _real_extract(self, url):
-        list_id = self._match_id(url)
-        webpage = self._download_webpage(url, list_id)
-
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
-
-        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
-        if pubdate:
-           pubdate = pubdate.replace('-','')
-
-        ret = []
-        # works with bbc.com/news/something-something-123456 articles
-        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
-        if not matches:
-           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-           # in http://www.bbc.com/news/video_and_audio/international
-           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-        if not matches:
-           raise ExtractorError('No video found', expected=True)
-
-        for ent in matches:
-            jent = self._parse_json(ent,list_id)
-
-            programme_id = jent.get('externalId',None)
-            xml_url = jent.get('href', None)
-
-            title = jent['caption']
-            duration = parse_duration(jent.get('duration',None))
-            description = list_title + ' - ' + jent.get('caption','')
-            thumbnail = None
-            if jent.has_key('image'):
-               thumbnail=jent['image'].get('href',None)
-
-            if programme_id:
-               formats, subtitles = self._download_media_selector(programme_id)
-            elif xml_url:
-               # Cheap fallback
-               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-               formats, subtitles = self._download_media_selector(programme_id)
-            else:
-               raise ExtractorError('data-media-meta entry has no externalId or href value.')
-               
-            self._sort_formats(formats)
-
-            ret.append( {
-                'id': programme_id,
-                'uploader': 'BBC News',
-                'upload_date': pubdate,
-                'title': title,
-                'description': description,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'formats': formats,
-                'subtitles': subtitles,
-            } )
-
-        if len(ret) > 0:
-           return self.playlist_result(ret, list_id, list_title)
-        raise ExtractorError('No video found', expected=True)

From 75ab0ebcf593ec91a46d83e69854ffa313d33309 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:24:02 -0500
Subject: [PATCH 0450/2145] no .get('..',None)

---
 youtube_dl/extractor/bbccouk.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index ea682fb6f..de4d7f9c0 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -457,15 +457,15 @@ class BBCNewsIE(BBCCoUkIE):
         for ent in matches:
             jent = self._parse_json(ent,list_id)
 
-            programme_id = jent.get('externalId',None)
-            xml_url = jent.get('href', None)
+            programme_id = jent.get('externalId')
+            xml_url = jent.get('href')
 
             title = jent['caption']
-            duration = parse_duration(jent.get('duration',None))
+            duration = parse_duration(jent.get('duration')
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):
-               thumbnail=jent['image'].get('href',None)
+               thumbnail=jent['image'].get('href')
 
             if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)

From 77c975f536befbe89bf718e86282958d391d9ffe Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:28:14 -0500
Subject: [PATCH 0451/2145] typofix

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index de4d7f9c0..f9404f3fa 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -461,7 +461,7 @@ class BBCNewsIE(BBCCoUkIE):
             xml_url = jent.get('href')
 
             title = jent['caption']
-            duration = parse_duration(jent.get('duration')
+            duration = parse_duration(jent.get('duration'))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):

From de939d89eb83c851c6db66933e5fc0c401a1a679 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:04:46 -0500
Subject: [PATCH 0452/2145] Support BBC news in other languages,
 non-mediaselector videos

---
 youtube_dl/extractor/bbccouk.py | 87 +++++++++++++++++++++++++++------
 1 file changed, 73 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f9404f3fa..72e20857b 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -401,7 +401,7 @@ class BBCCoUkIE(InfoExtractor):
 class BBCNewsIE(BBCCoUkIE):
     IE_NAME = 'bbc.com'
     IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.com/.+?/(?P<id>[^/]+)$'
 
     mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
 
@@ -432,56 +432,115 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    },{
+        'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'NA',
+            'ext': 'mp4',
+            'title': 'YPG - Tel Abyad..n tamam. kontrol.m.zde',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    },{
+        'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
+        'note': 'Video',
+        'info_dict': {
+            'id': '39275083',
+            'ext': 'mp4',
+            'title': 'Honduras militariza sus hospitales por nuevo esc.ndalo de corrupci.n',
+            'duration': 87,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
 
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'list title')
 
         pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
         if pubdate:
            pubdate = pubdate.replace('-','')
 
         ret = []
+        jsent = []
+
         # works with bbc.com/news/something-something-123456 articles
-        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
-        if not matches:
+        jsent = map(
+           lambda m: self._parse_json(m,list_id),
+           re.findall(r"data-media-meta='({[^']+})'", webpage)
+        )
+
+        if len(jsent) == 0:
+           # http://www.bbc.com/news/video_and_audio/international
+           # and single-video articles
+           masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
+           if masset:
+              jmasset = self._parse_json(masset,list_id)
+              for key, val in jmasset.get('videos',{}).items():
+                  for skey, sval in val.items():
+                      sval['id'] = skey
+                      jsent.append(sval)
+
+        if len(jsent) == 0:
            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
            # in http://www.bbc.com/news/video_and_audio/international
-           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-        if not matches:
+           # prone to breaking if entries have sourceFiles list
+           jsent = map(
+               lambda m: self._parse_json(m,list_id),
+               re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+           )          
+
+        if len(jsent) == 0:
            raise ExtractorError('No video found', expected=True)
 
-        for ent in matches:
-            jent = self._parse_json(ent,list_id)
-
+        for jent in jsent:
             programme_id = jent.get('externalId')
-            xml_url = jent.get('href')
+            xml_url = jent.get('hxref')
+
+            title = jent.get('caption',list_title)
 
-            title = jent['caption']
             duration = parse_duration(jent.get('duration'))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):
                thumbnail=jent['image'].get('href')
 
+            formats = []
+            subtitles = []
+
             if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)
+            elif jent.has_key('sourceFiles'):
+               # mediaselector not used at
+               # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
+               for key, val in jent['sourceFiles'].items():
+                  formats.append( {
+                     'ext': val.get('encoding'),
+                     'url': val.get('url'),
+                     'filesize': int(val.get('filesize')),
+                     'format_id': key
+                  } )
             elif xml_url:
                # Cheap fallback
                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
                formats, subtitles = self._download_media_selector(programme_id)
-            else:
-               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+
+            if len(formats) == 0:
+               raise ExtractorError('unsupported json media entry.\n    '+str(jent)+'\n')
                
             self._sort_formats(formats)
 
             ret.append( {
-                'id': programme_id,
+                'id': jent.get('programme_id',jent.get('id')),
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From 7bb23aeca4e9076528e3d31d501a9a288dcd444c Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:08:13 -0500
Subject: [PATCH 0453/2145] rename bbccouk.py -> bbc.py

---
 youtube_dl/extractor/__init__.py            | 2 +-
 youtube_dl/extractor/{bbccouk.py => bbc.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{bbccouk.py => bbc.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f9f7bdfaf..a48346e60 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -35,7 +35,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE, BBCNewsIE
+from .bbc import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbc.py
similarity index 100%
rename from youtube_dl/extractor/bbccouk.py
rename to youtube_dl/extractor/bbc.py

From 2a282a3b5f366ba0569bae477d5060329ba254fb Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:11:41 -0500
Subject: [PATCH 0454/2145] Unbreak breakage that was broken to test breakage

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 72e20857b..310db9d1d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -502,7 +502,7 @@ class BBCNewsIE(BBCCoUkIE):
 
         for jent in jsent:
             programme_id = jent.get('externalId')
-            xml_url = jent.get('hxref')
+            xml_url = jent.get('href')
 
             title = jent.get('caption',list_title)
 

From 964afd0689bdd7140b8ab182273d6379fe7b0548 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 03:12:17 +0800
Subject: [PATCH 0455/2145] [xvideos] Support lower-quality formats found on
 Android

Closes #5968
---
 youtube_dl/extractor/xvideos.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 2a45dc574..d8415bed4 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -5,10 +5,12 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     clean_html,
     ExtractorError,
+    determine_ext,
 )
 
 
@@ -25,6 +27,8 @@ class XVideosIE(InfoExtractor):
         }
     }
 
+    _ANDROID_USER_AGENT = 'Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -40,9 +44,30 @@ class XVideosIE(InfoExtractor):
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
 
+        formats = [{
+            'url': video_url,
+        }]
+
+        android_req = compat_urllib_request.Request(url)
+        android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
+        android_webpage = self._download_webpage(android_req, video_id, fatal=False)
+
+        if android_webpage is not None:
+            player_params_str = self._search_regex(
+                'mobileReplacePlayerDivTwoQual\(([^)]+)\)',
+                android_webpage, 'player parameters', default='')
+            player_params = list(map(lambda s: s.strip(' \''), player_params_str.split(',')))
+            if player_params:
+                formats.extend([{
+                    'url': param,
+                    'preference': -10,
+                } for param in player_params if determine_ext(param) == 'mp4'])
+
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': video_title,
             'ext': 'flv',
             'thumbnail': video_thumbnail,

From c9ac7fa909fb969ac21a6d168d09803119b018c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 04:17:54 +0600
Subject: [PATCH 0456/2145] [imdb] Fix extraction

---
 youtube_dl/extractor/imdb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f29df36b5..4bb574cf3 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -46,7 +46,7 @@ class ImdbIE(InfoExtractor):
             format_info = info['videoPlayerObject']['video']
             formats.append({
                 'format_id': f_id,
-                'url': format_info['url'],
+                'url': format_info['videoInfoList'][0]['videoUrl'],
             })
 
         return {

From af1fa6234e9623a1a09e66c24c0dedc6996645c9 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 21 Jun 2015 11:12:31 +0800
Subject: [PATCH 0457/2145] [neteasemusic] Add new extractor for music.163.com

---
 youtube_dl/extractor/__init__.py     |   9 +
 youtube_dl/extractor/neteasemusic.py | 434 +++++++++++++++++++++++++++
 2 files changed, 443 insertions(+)
 create mode 100644 youtube_dl/extractor/neteasemusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c548d8e9..c8692ffa9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -339,6 +339,15 @@ from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .nerdist import NerdistIE
+from .neteasemusic import (
+    NetEaseMusicIE,
+    NetEaseMusicAlbumIE,
+    NetEaseMusicSingerIE,
+    NetEaseMusicListIE,
+    NetEaseMusicMvIE,
+    NetEaseMusicProgramIE,
+    NetEaseMusicDjRadioIE,
+)
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nextmedia import (
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
new file mode 100644
index 000000000..ebed5d3b4
--- /dev/null
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -0,0 +1,434 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from hashlib import md5
+from datetime import datetime
+import itertools
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+
+
+class NetEaseMusicBaseIE(InfoExtractor):
+    _FORMATS = ['bMusic', 'mMusic', 'hMusic']
+    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
+    _API_BASE = 'http://music.163.com/api/'
+
+    @classmethod
+    def _encrypt(cls, dfsid):
+        salt_bytes = bytearray(str(cls._NETEASE_SALT))
+        string_bytes = bytearray(str(dfsid))
+        salt_len = len(salt_bytes)
+        for i in xrange(len(string_bytes)):
+            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
+        m = md5()
+        m.update(string_bytes)
+        result = m.digest().encode('base64')[:-1]
+        return result.replace('/', '_').replace('+', '-')
+
+    @classmethod
+    def extract_formats(cls, info):
+        formats = []
+        for song_format in cls._FORMATS:
+            details = info.get(song_format)
+            if not details:
+                continue
+            formats.append({
+                'url': 'http://m1.music.126.net/%s/%s.%s' %
+                       (cls._encrypt(details['dfsId']), details['dfsId'],
+                        details['extension']),
+                'ext': details['extension'],
+                'abr': details['bitrate'] / 1000,
+                'preference': details['bitrate'],
+                'format_id': song_format,
+                'filesize': details['size'],
+                'asr': details['sr']
+            })
+        return formats
+
+    def query_api(self, endpoint, video_id, note):
+        req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+        req.add_header('Referer', self._API_BASE)
+        return self._download_json(req, video_id, note)
+
+
+class NetEaseMusicIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:song'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/song?id=32102397',
+        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
+        'info_dict': {
+            'id': '32102397',
+            'ext': 'mp3',
+            'title': 'Bad Blood (feat. Kendrick Lamar)',
+            'creator': 'Taylor Swift / Kendrick Lamar',
+            'upload_date': '20150517',
+            'timestamp': 1431878400,
+            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
+        },
+    }, {
+        'note': 'No lyrics translation.',
+        'url': 'http://music.163.com/#/song?id=29822014',
+        'info_dict': {
+            'id': '29822014',
+            'ext': 'mp3',
+            'title': '听见下雨的声音',
+            'creator': '周杰伦',
+            'upload_date': '20141225',
+            'timestamp': 1419523200,
+            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
+        },
+    }, {
+        'note': 'No lyrics.',
+        'url': 'http://music.163.com/song?id=17241424',
+        'info_dict': {
+            'id': '17241424',
+            'ext': 'mp3',
+            'title': 'Opus 28',
+            'creator': 'Dustin O\'Halloran',
+            'upload_date': '20080211',
+            'timestamp': 1202745600,
+        },
+    }]
+
+    def _process_lyrics(self, lyrics_info):
+        original = lyrics_info.get('lrc', {}).get('lyric')
+        translated = lyrics_info.get('tlyric', {}).get('lyric')
+
+        if not translated:
+            return original
+
+        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
+        original_ts_texts = re.findall(lyrics_expr, original)
+        translation_ts_dict = {
+            time_stamp: text for time_stamp, text in re.findall(lyrics_expr, translated)
+        }
+
+        lyrics = '\n'.join([
+            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
+            for time_stamp, text in original_ts_texts
+        ])
+        return lyrics
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+
+        params = {
+            'id': song_id,
+            'ids': '[%s]' % song_id
+        }
+        info = self.query_api(
+            'song/detail?' + compat_urllib_parse.urlencode(params),
+            song_id, 'Downloading song info')['songs'][0]
+
+        formats = self.extract_formats(info)
+        self._sort_formats(formats)
+
+        lyrics_info = self.query_api(
+            'song/lyric?id=%s&lv=-1&tv=-1' % song_id, 
+            song_id, 'Downloading lyrics data')
+        lyrics = self._process_lyrics(lyrics_info)
+
+        alt_title = None
+        if info.get('alias'):
+            alt_title = '/'.join(info.get('alias'))
+
+        return {
+            'id': song_id,
+            'title': info['name'],
+            'alt_title': alt_title,
+            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
+            'timestamp': int(info.get('album', {}).get('publishTime')/1000),
+            'thumbnail': info.get('album', {}).get('picUrl'),
+            'duration': int(info.get('duration', 0)/1000),
+            'description': lyrics,
+            'formats': formats,
+        }
+
+
+class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:album'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/album?id=220780',
+        'info_dict': {
+            'id': '220780',
+            'title': 'B\'day',
+        },
+        'playlist_count': 23,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        info = self.query_api(
+            'album/%s?id=%s' % (album_id, album_id), 
+            album_id, 'Downloading album data')['album']
+
+        name = info['name']
+        desc = info.get('description')
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['songs']
+        ]
+        return self.playlist_result(entries, album_id, name, desc)
+
+
+class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:singer'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'note': 'Singer has aliases.',
+        'url': 'http://music.163.com/#/artist?id=10559',
+        'info_dict': {
+            'id': '10559',
+            'title': '张惠妹 - aMEI;阿密特',
+        },
+        'playlist_count': 50,
+    }, {
+        'note': 'Singer has translated name.',
+        'url': 'http://music.163.com/#/artist?id=124098',
+        'info_dict': {
+            'id': '124098',
+            'title': '李昇基 - 이승기',
+        },
+        'playlist_count': 50,
+    }]
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+
+        info = self.query_api(
+            'artist/%s?id=%s' % (singer_id, singer_id), 
+            singer_id, 'Downloading singer data')
+
+        name = info['artist']['name']
+        if info['artist']['trans']:
+            name = '%s - %s' % (name, info['artist']['trans'])
+        if info['artist']['alias']:
+            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['hotSongs']
+        ]
+        return self.playlist_result(entries, singer_id, name)
+
+
+class NetEaseMusicListIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:playlist'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/playlist?id=79177352',
+        'info_dict': {
+            'id': '79177352',
+            'title': 'Billboard 2007 Top 100',
+            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
+        },
+        'playlist_count': 99,
+    }, {
+        'note': 'Toplist/Charts sample',
+        'url': 'http://music.163.com/#/discover/toplist?id=3733003',
+        'info_dict': {
+            'id': '3733003',
+            'title': 're:韩国Melon排行榜周榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
+            'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
+        },
+        'playlist_count': 50,
+    }]
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        info = self.query_api(
+            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id, 
+            list_id, 'Downloading playlist data')['result']
+
+        name = info['name']
+        desc = info.get('description')
+
+        if info.get('specialType') == 10:  # is a chart/toplist
+            datestamp = datetime.fromtimestamp(info['updateTime']/1000).strftime('%Y-%m-%d')
+            name = '%s %s' % (name, datestamp)
+
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['tracks']
+        ]
+        return self.playlist_result(entries, list_id, name, desc)
+
+
+class NetEaseMusicMvIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:mv'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/mv?id=415350',
+        'info_dict': {
+            'id': '415350',
+            'ext': 'mp4',
+            'title': '이럴거면 그러지말지',
+            'description': '白雅言自作曲唱甜蜜爱情',
+            'creator': '白雅言',
+            'upload_date': '20150520',
+        },
+    }
+
+    def _real_extract(self, url):
+        mv_id = self._match_id(url)
+
+        info = self.query_api(
+            'mv/detail?id=%s&type=mp4' % mv_id,
+            mv_id, 'Downloading mv info')['data']
+
+        formats = [
+            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'preference': int(brs)}
+            for brs, mv_url in info['brs'].items()
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': mv_id,
+            'title': info['name'],
+            'description': info.get('desc') or info.get('briefDesc'),
+            'creator': info['artistName'],
+            'upload_date': info['publishTime'].replace('-', ''),
+            'formats': formats,
+            'thumbnail': info.get('cover'),
+            'duration': int(info.get('duration', 0)/1000),
+        }
+
+
+class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:program'
+    _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/program?id=10109055',
+        'info_dict': {
+            'id': '10109055',
+            'ext': 'mp3',
+            'title': '不丹足球背后的故事',
+            'description': '喜马拉雅人的足球梦 ...',
+            'creator': '大话西藏',
+            'timestamp': 1434179341,
+            'upload_date': '20150613',
+            'duration': 900,
+        },
+    }, {
+        'note': 'This program has accompanying songs.',
+        'url': 'http://music.163.com/#/program?id=10141022',
+        'info_dict': {
+            'id': '10141022',
+            'title': '25岁，你是自在如风的少年<27°C>',
+            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+        },
+        'playlist_count': 4,
+    }, {
+        'note': 'This program has accompanying songs.',
+        'url': 'http://music.163.com/#/program?id=10141022',
+        'info_dict': {
+            'id': '10141022',
+            'ext': 'mp3',
+            'title': '25岁，你是自在如风的少年<27°C>',
+            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+            'timestamp': 1434450840,
+            'upload_date': '20150616',
+        },
+        'params': {
+            'noplaylist': True
+        }
+    }]
+
+    def _real_extract(self, url):
+        program_id = self._match_id(url)
+
+        info = self.query_api(
+            'dj/program/detail?id=%s' % program_id,
+            program_id, 'Downloading program info')['program']
+
+        name = info['name']
+        description = info['description']
+
+        if not info['songs'] or self._downloader.params.get('noplaylist'):
+            if info['songs']:
+                self.to_screen(
+                    'Downloading just the main audio %s because of --no-playlist'
+                    % info['mainSong']['id'])
+            
+            formats = self.extract_formats(info['mainSong'])
+            self._sort_formats(formats)
+
+            return {
+                'id': program_id,
+                'title': name,
+                'description': description,
+                'creator': info['dj']['brand'],
+                'timestamp': int(info['createTime']/1000),
+                'thumbnail': info['coverUrl'],
+                'duration': int(info.get('duration', 0)/1000),
+                'formats': formats,
+            }
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to just download the main audio %s'
+            % (program_id, info['mainSong']['id']))
+
+        song_ids = [info['mainSong']['id']]
+        song_ids.extend([song['id'] for song in info['songs']])
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
+                            'NetEaseMusic', song_id)
+            for song_id in song_ids
+        ]
+        return self.playlist_result(entries, program_id, name, description)
+
+
+class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:djradio'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/djradio?id=42',
+        'info_dict': {
+            'id': '42',
+            'title': '声音蔓延',
+            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
+        },
+        'playlist_mincount': 40,
+    }
+    _PAGE_SIZE = 1000
+
+    def _real_extract(self, url):
+        dj_id = self._match_id(url)
+
+        name = None
+        desc = None
+        entries = []
+        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+            info = self.query_api(
+                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d' 
+                % (self._PAGE_SIZE, dj_id, offset),
+                dj_id, 'Downloading dj programs - %d' % offset)
+
+            entries.extend([
+                self.url_result(
+                    'http://music.163.com/#/program?id=%s' % program['id'],
+                    'NetEaseMusicProgram', program['id'])
+                for program in info['programs']
+            ])
+
+            if name is None:
+                radio = info['programs'][0]['radio']
+                name = radio['name']
+                desc = radio['desc']
+
+            if not info['more']:
+                break
+
+        return self.playlist_result(entries, dj_id, name, desc)

From 2da0cad6ae08e3c78f85a340260aef80a464cb20 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 21 Jun 2015 11:44:50 +0800
Subject: [PATCH 0458/2145] [neteasemusic] Do proper rounding conversion of
 millisecond timestamps/durations

---
 youtube_dl/extractor/neteasemusic.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index ebed5d3b4..a70c65ca5 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -50,6 +50,10 @@ class NetEaseMusicBaseIE(InfoExtractor):
             })
         return formats
 
+    @classmethod
+    def convert_milliseconds(cls, ms):
+        return int(round(ms/1000.0))
+
     def query_api(self, endpoint, video_id, note):
         req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
         req.add_header('Referer', self._API_BASE)
@@ -143,9 +147,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': info['name'],
             'alt_title': alt_title,
             'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
-            'timestamp': int(info.get('album', {}).get('publishTime')/1000),
+            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
             'thumbnail': info.get('album', {}).get('picUrl'),
-            'duration': int(info.get('duration', 0)/1000),
+            'duration': self.convert_milliseconds(info.get('duration', 0)),
             'description': lyrics,
             'formats': formats,
         }
@@ -255,7 +259,8 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         desc = info.get('description')
 
         if info.get('specialType') == 10:  # is a chart/toplist
-            datestamp = datetime.fromtimestamp(info['updateTime']/1000).strftime('%Y-%m-%d')
+            datestamp = datetime.fromtimestamp(
+                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
             name = '%s %s' % (name, datestamp)
 
         entries = [
@@ -302,7 +307,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             'upload_date': info['publishTime'].replace('-', ''),
             'formats': formats,
             'thumbnail': info.get('cover'),
-            'duration': int(info.get('duration', 0)/1000),
+            'duration': self.convert_milliseconds(info.get('duration', 0)),
         }
 
 
@@ -317,7 +322,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'title': '不丹足球背后的故事',
             'description': '喜马拉雅人的足球梦 ...',
             'creator': '大话西藏',
-            'timestamp': 1434179341,
+            'timestamp': 1434179342,
             'upload_date': '20150613',
             'duration': 900,
         },
@@ -338,7 +343,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'ext': 'mp3',
             'title': '25岁，你是自在如风的少年<27°C>',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
-            'timestamp': 1434450840,
+            'timestamp': 1434450841,
             'upload_date': '20150616',
         },
         'params': {
@@ -370,9 +375,9 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
                 'title': name,
                 'description': description,
                 'creator': info['dj']['brand'],
-                'timestamp': int(info['createTime']/1000),
+                'timestamp': self.convert_milliseconds(info['createTime']),
                 'thumbnail': info['coverUrl'],
-                'duration': int(info.get('duration', 0)/1000),
+                'duration': self.convert_milliseconds(info.get('duration', 0)),
                 'formats': formats,
             }
 

From 6a745c2c0fa2d627b46f2d4d8013fa69276c4fac Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:22:19 +0800
Subject: [PATCH 0459/2145] [pinkbike] PEP8

---
 youtube_dl/extractor/pinkbike.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 745433b48..a52210fab 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -44,7 +44,7 @@ class PinkbikeIE(InfoExtractor):
 
         formats = []
         for _, format_id, src in re.findall(
-            r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
+                r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
             height = int_or_none(self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             formats.append({

From b407e173e44041b1a92fb61e316f92d19834a40a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:23:58 +0800
Subject: [PATCH 0460/2145] [vimeo/generic] Move detection logic from GenericIE
 to VimeoIE

---
 youtube_dl/extractor/generic.py | 16 ++++------------
 youtube_dl/extractor/vimeo.py   | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f6b984300..bf689f531 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -43,6 +43,7 @@ from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
+from .vimeo import VimeoIE
 
 
 class GenericIE(InfoExtractor):
@@ -1089,18 +1090,9 @@ class GenericIE(InfoExtractor):
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')
 
-        # Look for embedded (iframe) Vimeo player
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
-        if mobj:
-            player_url = unescapeHTML(mobj.group('url'))
-            surl = smuggle_url(player_url, {'Referer': url})
-            return self.url_result(surl)
-        # Look for embedded (swf embed) Vimeo player
-        mobj = re.search(
-            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
-        if mobj:
-            return self.url_result(mobj.group(1))
+        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
+        if vimeo_url is not None:
+            return self.url_result(vimeo_url)
 
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index f300c7ca4..cae90205d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -22,6 +22,7 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     urlencode_postdata,
+    unescapeHTML,
 )
 
 
@@ -173,6 +174,21 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
     ]
 
+    @staticmethod
+    def _extract_vimeo_url(url, webpage):
+        # Look for embedded (iframe) Vimeo player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
+        if mobj:
+            player_url = unescapeHTML(mobj.group('url'))
+            surl = smuggle_url(player_url, {'Referer': url})
+            return surl
+        # Look for embedded (swf embed) Vimeo player
+        mobj = re.search(
+            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
+        if mobj:
+            return mobj.group(1)
+
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:

From c5895d5dbdc33fbad1c91f448704d7711448220d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:30:38 +0800
Subject: [PATCH 0461/2145] [tumblr] Support Vimeo embeds (fixes #5969)

---
 youtube_dl/extractor/tumblr.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 63c20310d..9ead13a91 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from .pornhub import PornHubIE
+from .vimeo import VimeoIE
 
 
 class TumblrIE(InfoExtractor):
@@ -40,6 +41,17 @@ class TumblrIE(InfoExtractor):
             'timestamp': 1430931613,
         },
         'add_ie': ['Vidme'],
+    }, {
+        'url': 'http://camdamage.tumblr.com/post/98846056295/',
+        'md5': 'a9e0c8371ea1ca306d6554e3fecf50b6',
+        'info_dict': {
+            'id': '105463834',
+            'ext': 'mp4',
+            'title': 'Cam Damage-HD 720p',
+            'uploader': 'John Moyer',
+            'uploader_id': 'user32021558',
+        },
+        'add_ie': ['Vimeo'],
     }]
 
     def _real_extract(self, url):
@@ -60,6 +72,10 @@ class TumblrIE(InfoExtractor):
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
 
+        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
+        if vimeo_url:
+            return self.url_result(vimeo_url, 'Vimeo')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From 396726244a9096f142f5420ba5f3a1a36abb9a86 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:53:17 +0800
Subject: [PATCH 0462/2145] [utils/ffmpeg] Move ISO 639 related codes to utils

---
 youtube_dl/postprocessor/ffmpeg.py | 196 +---------------------------
 youtube_dl/utils.py                | 202 +++++++++++++++++++++++++++++
 2 files changed, 204 insertions(+), 194 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index cc65b34e7..fe7e0a8ee 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -21,6 +21,7 @@ from ..utils import (
     shell_quote,
     subtitles_filename,
     dfxp2srt,
+    ISO639Utils,
 )
 
 
@@ -307,199 +308,6 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
-    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-    _lang_map = {
-        'aa': 'aar',
-        'ab': 'abk',
-        'ae': 'ave',
-        'af': 'afr',
-        'ak': 'aka',
-        'am': 'amh',
-        'an': 'arg',
-        'ar': 'ara',
-        'as': 'asm',
-        'av': 'ava',
-        'ay': 'aym',
-        'az': 'aze',
-        'ba': 'bak',
-        'be': 'bel',
-        'bg': 'bul',
-        'bh': 'bih',
-        'bi': 'bis',
-        'bm': 'bam',
-        'bn': 'ben',
-        'bo': 'bod',
-        'br': 'bre',
-        'bs': 'bos',
-        'ca': 'cat',
-        'ce': 'che',
-        'ch': 'cha',
-        'co': 'cos',
-        'cr': 'cre',
-        'cs': 'ces',
-        'cu': 'chu',
-        'cv': 'chv',
-        'cy': 'cym',
-        'da': 'dan',
-        'de': 'deu',
-        'dv': 'div',
-        'dz': 'dzo',
-        'ee': 'ewe',
-        'el': 'ell',
-        'en': 'eng',
-        'eo': 'epo',
-        'es': 'spa',
-        'et': 'est',
-        'eu': 'eus',
-        'fa': 'fas',
-        'ff': 'ful',
-        'fi': 'fin',
-        'fj': 'fij',
-        'fo': 'fao',
-        'fr': 'fra',
-        'fy': 'fry',
-        'ga': 'gle',
-        'gd': 'gla',
-        'gl': 'glg',
-        'gn': 'grn',
-        'gu': 'guj',
-        'gv': 'glv',
-        'ha': 'hau',
-        'he': 'heb',
-        'hi': 'hin',
-        'ho': 'hmo',
-        'hr': 'hrv',
-        'ht': 'hat',
-        'hu': 'hun',
-        'hy': 'hye',
-        'hz': 'her',
-        'ia': 'ina',
-        'id': 'ind',
-        'ie': 'ile',
-        'ig': 'ibo',
-        'ii': 'iii',
-        'ik': 'ipk',
-        'io': 'ido',
-        'is': 'isl',
-        'it': 'ita',
-        'iu': 'iku',
-        'ja': 'jpn',
-        'jv': 'jav',
-        'ka': 'kat',
-        'kg': 'kon',
-        'ki': 'kik',
-        'kj': 'kua',
-        'kk': 'kaz',
-        'kl': 'kal',
-        'km': 'khm',
-        'kn': 'kan',
-        'ko': 'kor',
-        'kr': 'kau',
-        'ks': 'kas',
-        'ku': 'kur',
-        'kv': 'kom',
-        'kw': 'cor',
-        'ky': 'kir',
-        'la': 'lat',
-        'lb': 'ltz',
-        'lg': 'lug',
-        'li': 'lim',
-        'ln': 'lin',
-        'lo': 'lao',
-        'lt': 'lit',
-        'lu': 'lub',
-        'lv': 'lav',
-        'mg': 'mlg',
-        'mh': 'mah',
-        'mi': 'mri',
-        'mk': 'mkd',
-        'ml': 'mal',
-        'mn': 'mon',
-        'mr': 'mar',
-        'ms': 'msa',
-        'mt': 'mlt',
-        'my': 'mya',
-        'na': 'nau',
-        'nb': 'nob',
-        'nd': 'nde',
-        'ne': 'nep',
-        'ng': 'ndo',
-        'nl': 'nld',
-        'nn': 'nno',
-        'no': 'nor',
-        'nr': 'nbl',
-        'nv': 'nav',
-        'ny': 'nya',
-        'oc': 'oci',
-        'oj': 'oji',
-        'om': 'orm',
-        'or': 'ori',
-        'os': 'oss',
-        'pa': 'pan',
-        'pi': 'pli',
-        'pl': 'pol',
-        'ps': 'pus',
-        'pt': 'por',
-        'qu': 'que',
-        'rm': 'roh',
-        'rn': 'run',
-        'ro': 'ron',
-        'ru': 'rus',
-        'rw': 'kin',
-        'sa': 'san',
-        'sc': 'srd',
-        'sd': 'snd',
-        'se': 'sme',
-        'sg': 'sag',
-        'si': 'sin',
-        'sk': 'slk',
-        'sl': 'slv',
-        'sm': 'smo',
-        'sn': 'sna',
-        'so': 'som',
-        'sq': 'sqi',
-        'sr': 'srp',
-        'ss': 'ssw',
-        'st': 'sot',
-        'su': 'sun',
-        'sv': 'swe',
-        'sw': 'swa',
-        'ta': 'tam',
-        'te': 'tel',
-        'tg': 'tgk',
-        'th': 'tha',
-        'ti': 'tir',
-        'tk': 'tuk',
-        'tl': 'tgl',
-        'tn': 'tsn',
-        'to': 'ton',
-        'tr': 'tur',
-        'ts': 'tso',
-        'tt': 'tat',
-        'tw': 'twi',
-        'ty': 'tah',
-        'ug': 'uig',
-        'uk': 'ukr',
-        'ur': 'urd',
-        'uz': 'uzb',
-        've': 'ven',
-        'vi': 'vie',
-        'vo': 'vol',
-        'wa': 'wln',
-        'wo': 'wol',
-        'xh': 'xho',
-        'yi': 'yid',
-        'yo': 'yor',
-        'za': 'zha',
-        'zh': 'zho',
-        'zu': 'zul',
-    }
-
-    @classmethod
-    def _conver_lang_code(cls, code):
-        """Convert language code from ISO 639-1 to ISO 639-2/T"""
-        return cls._lang_map.get(code[:2])
-
     def run(self, information):
         if information['ext'] not in ['mp4', 'mkv']:
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
@@ -525,7 +333,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             opts += ['-c:s', 'mov_text']
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1)])
-            lang_code = self._conver_lang_code(lang)
+            lang_code = ISO639Utils.short2long(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52d198fa3..259a9d634 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1879,6 +1879,208 @@ def dfxp2srt(dfxp_data):
     return ''.join(out)
 
 
+class ISO639Utils(object):
+    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
+    _lang_map = {
+        'aa': 'aar',
+        'ab': 'abk',
+        'ae': 'ave',
+        'af': 'afr',
+        'ak': 'aka',
+        'am': 'amh',
+        'an': 'arg',
+        'ar': 'ara',
+        'as': 'asm',
+        'av': 'ava',
+        'ay': 'aym',
+        'az': 'aze',
+        'ba': 'bak',
+        'be': 'bel',
+        'bg': 'bul',
+        'bh': 'bih',
+        'bi': 'bis',
+        'bm': 'bam',
+        'bn': 'ben',
+        'bo': 'bod',
+        'br': 'bre',
+        'bs': 'bos',
+        'ca': 'cat',
+        'ce': 'che',
+        'ch': 'cha',
+        'co': 'cos',
+        'cr': 'cre',
+        'cs': 'ces',
+        'cu': 'chu',
+        'cv': 'chv',
+        'cy': 'cym',
+        'da': 'dan',
+        'de': 'deu',
+        'dv': 'div',
+        'dz': 'dzo',
+        'ee': 'ewe',
+        'el': 'ell',
+        'en': 'eng',
+        'eo': 'epo',
+        'es': 'spa',
+        'et': 'est',
+        'eu': 'eus',
+        'fa': 'fas',
+        'ff': 'ful',
+        'fi': 'fin',
+        'fj': 'fij',
+        'fo': 'fao',
+        'fr': 'fra',
+        'fy': 'fry',
+        'ga': 'gle',
+        'gd': 'gla',
+        'gl': 'glg',
+        'gn': 'grn',
+        'gu': 'guj',
+        'gv': 'glv',
+        'ha': 'hau',
+        'he': 'heb',
+        'hi': 'hin',
+        'ho': 'hmo',
+        'hr': 'hrv',
+        'ht': 'hat',
+        'hu': 'hun',
+        'hy': 'hye',
+        'hz': 'her',
+        'ia': 'ina',
+        'id': 'ind',
+        'ie': 'ile',
+        'ig': 'ibo',
+        'ii': 'iii',
+        'ik': 'ipk',
+        'io': 'ido',
+        'is': 'isl',
+        'it': 'ita',
+        'iu': 'iku',
+        'ja': 'jpn',
+        'jv': 'jav',
+        'ka': 'kat',
+        'kg': 'kon',
+        'ki': 'kik',
+        'kj': 'kua',
+        'kk': 'kaz',
+        'kl': 'kal',
+        'km': 'khm',
+        'kn': 'kan',
+        'ko': 'kor',
+        'kr': 'kau',
+        'ks': 'kas',
+        'ku': 'kur',
+        'kv': 'kom',
+        'kw': 'cor',
+        'ky': 'kir',
+        'la': 'lat',
+        'lb': 'ltz',
+        'lg': 'lug',
+        'li': 'lim',
+        'ln': 'lin',
+        'lo': 'lao',
+        'lt': 'lit',
+        'lu': 'lub',
+        'lv': 'lav',
+        'mg': 'mlg',
+        'mh': 'mah',
+        'mi': 'mri',
+        'mk': 'mkd',
+        'ml': 'mal',
+        'mn': 'mon',
+        'mr': 'mar',
+        'ms': 'msa',
+        'mt': 'mlt',
+        'my': 'mya',
+        'na': 'nau',
+        'nb': 'nob',
+        'nd': 'nde',
+        'ne': 'nep',
+        'ng': 'ndo',
+        'nl': 'nld',
+        'nn': 'nno',
+        'no': 'nor',
+        'nr': 'nbl',
+        'nv': 'nav',
+        'ny': 'nya',
+        'oc': 'oci',
+        'oj': 'oji',
+        'om': 'orm',
+        'or': 'ori',
+        'os': 'oss',
+        'pa': 'pan',
+        'pi': 'pli',
+        'pl': 'pol',
+        'ps': 'pus',
+        'pt': 'por',
+        'qu': 'que',
+        'rm': 'roh',
+        'rn': 'run',
+        'ro': 'ron',
+        'ru': 'rus',
+        'rw': 'kin',
+        'sa': 'san',
+        'sc': 'srd',
+        'sd': 'snd',
+        'se': 'sme',
+        'sg': 'sag',
+        'si': 'sin',
+        'sk': 'slk',
+        'sl': 'slv',
+        'sm': 'smo',
+        'sn': 'sna',
+        'so': 'som',
+        'sq': 'sqi',
+        'sr': 'srp',
+        'ss': 'ssw',
+        'st': 'sot',
+        'su': 'sun',
+        'sv': 'swe',
+        'sw': 'swa',
+        'ta': 'tam',
+        'te': 'tel',
+        'tg': 'tgk',
+        'th': 'tha',
+        'ti': 'tir',
+        'tk': 'tuk',
+        'tl': 'tgl',
+        'tn': 'tsn',
+        'to': 'ton',
+        'tr': 'tur',
+        'ts': 'tso',
+        'tt': 'tat',
+        'tw': 'twi',
+        'ty': 'tah',
+        'ug': 'uig',
+        'uk': 'ukr',
+        'ur': 'urd',
+        'uz': 'uzb',
+        've': 'ven',
+        'vi': 'vie',
+        'vo': 'vol',
+        'wa': 'wln',
+        'wo': 'wol',
+        'xh': 'xho',
+        'yi': 'yid',
+        'yo': 'yor',
+        'za': 'zha',
+        'zh': 'zho',
+        'zu': 'zul',
+    }
+
+    @classmethod
+    def short2long(cls, code):
+        """Convert language code from ISO 639-1 to ISO 639-2/T"""
+        return cls._lang_map.get(code[:2])
+
+    @classmethod
+    def long2short(cls, code):
+        """Convert language code from ISO 639-2/T to ISO 639-1"""
+        for short_name, long_name in cls._lang_map.items():
+            if long_name == code:
+                return short_name
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From 607841af64d308eaf577e528fd7317a8b382b8e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:55:26 +0800
Subject: [PATCH 0463/2145] [adobetv] Support embeddable videos (closes #6039)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/adobetv.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c548d8e9..0f4af88f0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -4,7 +4,10 @@ from .abc import ABCIE
 from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
-from .adobetv import AdobeTVIE
+from .adobetv import (
+    AdobeTVIE,
+    AdobeTVVideoIE,
+)
 from .adultswim import AdultSwimIE
 from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 97d128560..695a4a15c 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -5,6 +5,8 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     str_to_int,
+    float_or_none,
+    ISO639Utils,
 )
 
 
@@ -69,3 +71,60 @@ class AdobeTVIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
         }
+
+
+class AdobeTVVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'https://video.tv.adobe.com/v/2456/',
+        'md5': '43662b577c018ad707a63766462b1e87',
+        'info_dict': {
+            'id': '2456',
+            'ext': 'mp4',
+            'title': 'New experience with Acrobat DC',
+            'description': 'New experience with Acrobat DC',
+            'duration': 248.667,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player_params = self._parse_json(self._search_regex(
+            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'player parameters'),
+            video_id)
+
+        formats = [{
+            'url': source['src'],
+            'width': source.get('width'),
+            'height': source.get('height'),
+            'tbr': source.get('bitrate'),
+        } for source in player_params['sources']]
+
+        # For both metadata and downloaded files the duration varies among
+        # formats. I just pick the max one
+        duration = max(filter(None, [
+            float_or_none(source.get('duration'), scale=1000)
+            for source in player_params['sources']]))
+
+        subtitles = {}
+        for translation in player_params.get('translations', []):
+            lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
+            if lang_id not in subtitles:
+                subtitles[lang_id] = []
+            subtitles[lang_id].append({
+                'url': translation['vttPath'],
+                'ext': 'vtt',
+            })
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': player_params['title'],
+            'description': self._og_search_description(webpage),
+            'duration': duration,
+            'subtitles': subtitles,
+        }

From 4e3357717312ac56145ba166a1ae2806f6db8337 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 19:16:59 +0800
Subject: [PATCH 0464/2145] [utils] Support ttaf1 namespace in TTML

It's found in bbc.co.uk. See #6038
---
 youtube_dl/utils.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 259a9d634..a2746b2d1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1841,7 +1841,10 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
-    _x = functools.partial(xpath_with_ns, ns_map={'ttml': 'http://www.w3.org/ns/ttml'})
+    _x = functools.partial(xpath_with_ns, ns_map={
+        'ttml': 'http://www.w3.org/ns/ttml',
+        'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
+    })
 
     def parse_node(node):
         str_or_empty = functools.partial(str_or_none, default='')
@@ -1849,9 +1852,9 @@ def dfxp2srt(dfxp_data):
         out = str_or_empty(node.text)
 
         for child in node:
-            if child.tag in (_x('ttml:br'), 'br'):
+            if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
                 out += '\n' + str_or_empty(child.tail)
-            elif child.tag in (_x('ttml:span'), 'span'):
+            elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
                 out += str_or_empty(parse_node(child))
             else:
                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
@@ -1860,7 +1863,7 @@ def dfxp2srt(dfxp_data):
 
     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')

From 78294e6a9ce2c9a294d663ac79936df7353b9980 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 19:22:26 +0800
Subject: [PATCH 0465/2145] [bbccouk] Remove TTML to srt conversion codes

It's broken. See #6038
---
 youtube_dl/extractor/bbccouk.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 0305f88b5..5825d2867 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -251,26 +251,11 @@ class BBCCoUkIE(InfoExtractor):
         for connection in self._extract_connections(media):
             captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
-            ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
-            srt = ''
-
-            def _extract_text(p):
-                if p.text is not None:
-                    stripped_text = p.text.strip()
-                    if stripped_text:
-                        return stripped_text
-                return ' '.join(span.text.strip() for span in p.findall('{http://www.w3.org/2006/10/ttaf1}span'))
-            for pos, p in enumerate(ps):
-                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'), _extract_text(p))
             subtitles[lang] = [
                 {
                     'url': connection.get('href'),
                     'ext': 'ttml',
                 },
-                {
-                    'data': srt,
-                    'ext': 'srt',
-                },
             ]
         return subtitles
 

From 756f574e4e7160ca5b39c6e18ec5168beb4a8eb1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 21:30:34 +0800
Subject: [PATCH 0466/2145] [dailymotion/generic] Add DailymotionCloudIE

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/dailymotion.py | 42 +++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py     | 17 ++++++++++++
 3 files changed, 60 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0f4af88f0..bd3c3193f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
     DailymotionUserIE,
+    DailymotionCloudIE,
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 70aa4333c..96f0ed9ad 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -251,3 +251,45 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': full_user,
             'entries': self._extract_entries(user),
         }
+
+
+class DailymotionCloudIE(DailymotionBaseInfoExtractor):
+    _VALID_URL = r'http://api\.dmcloud\.net/embed/[^/]+/(?P<id>[^/?]+)'
+
+    _TEST = {
+        # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
+        # Tested at FranceTvInfo_2
+        'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
+        'only_matching': True,
+    }
+
+    @classmethod
+    def _extract_dmcloud_url(self, webpage):
+        mobj = re.search(r'<iframe[^>]+src=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        if mobj:
+            return mobj.group(1)
+
+        mobj = re.search(r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        if mobj:
+            return mobj.group(1)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        request = self._build_request(url)
+        webpage = self._download_webpage(request, video_id)
+
+        title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
+
+        video_info = self._parse_json(self._search_regex(
+            r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id)
+
+        # TODO: parse ios_url, which is in fact a manifest
+        video_url = video_info['mp4_url']
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': video_info.get('thumbnail_url'),
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bf689f531..07939b196 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -44,6 +44,7 @@ from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .vimeo import VimeoIE
+from .dailymotion import DailymotionCloudIE
 
 
 class GenericIE(InfoExtractor):
@@ -813,6 +814,17 @@ class GenericIE(InfoExtractor):
                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
                 'uploader': 'Rogers Sportsnet',
             },
+        },
+        # Dailymotion Cloud video
+        {
+            'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
+            'md5': '49444254273501a64675a7e68c502681',
+            'info_dict': {
+                'id': '5585de919473990de4bee11b',
+                'ext': 'mp4',
+                'title': 'Le débat',
+                'thumbnail': 're:^https?://.*\.jpe?g$',
+            }
         }
     ]
 
@@ -1486,6 +1498,11 @@ class GenericIE(InfoExtractor):
         if senate_isvp_url:
             return self.url_result(senate_isvp_url, 'SenateISVP')
 
+        # Look for Dailymotion Cloud videos
+        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
+        if dmcloud_url:
+            return self.url_result(dmcloud_url, 'DailymotionCloud')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 6f96e308d0fa7674ac88e1e80fc602413f9a6b31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 21:31:33 +0800
Subject: [PATCH 0467/2145] [francetvinfo.fr] Support dmcloud embeds (fixes
 #6034)

---
 youtube_dl/extractor/francetv.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index db0bbec1e..b2c984bf2 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -18,6 +18,7 @@ from ..utils import (
     parse_duration,
     determine_ext,
 )
+from .dailymotion import DailymotionCloudIE
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -131,12 +132,26 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'skip_download': 'HLS (reqires ffmpeg)'
         },
         'skip': 'Ce direct est terminé et sera disponible en rattrapage dans quelques minutes.',
+    }, {
+        'url': 'http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html',
+        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
+        'info_dict': {
+            'id': '556e03339473995ee145930c',
+            'ext': 'mp4',
+            'title': 'Les entreprises familiales : le secret de la réussite',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+        }
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
+
+        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
+        if dmcloud_url:
+            return self.url_result(dmcloud_url, 'DailymotionCloud')
+
         video_id, catalogue = self._search_regex(
             r'id-video=([^@]+@[^"]+)', webpage, 'video id').split('@')
         return self._extract_video(video_id, catalogue)

From 3f3308cd75fc068e4d67d00aa7d7892e02ab16e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 23:29:40 +0800
Subject: [PATCH 0468/2145] Revert "[sohu] Update extractor"

This reverts commit 32060c6d6b618fa858b2ce43db34d02fd43bc542.
---
 youtube_dl/extractor/sohu.py | 46 ++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 29bd9ce6f..7644cc02d 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import ExtractorError
+from ..utils import (
+    sanitize_url_path_consecutive_slashes,
+    ExtractorError,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -26,7 +29,7 @@ class SohuIE(InfoExtractor):
         'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',
+        'md5': '699060e75cf58858dd47fb9c03c42cfb',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -34,7 +37,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '49308ff6dafde5ece51137d04aec311e',
+        'md5': '9bf34be48f2f4dadcb226c74127e203c',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -48,7 +51,7 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': '492923eac023ba2f13ff69617c32754a',
+            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -56,7 +59,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': 'de604848c0e8e9c4a4dde7e1347c0637',
+            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -64,7 +67,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '93584716ee0657c0b205b8aa3d27aa13',
+            'md5': '8407e634175fdac706766481b9443450',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',
@@ -139,21 +142,24 @@ class SohuIE(InfoExtractor):
         for i in range(part_count):
             formats = []
             for format_id, format_data in formats_json.items():
-                data = format_data['data']
+                allot = format_data['allot']
+                prot = format_data['prot']
 
-                # URLs starts with http://newflv.sohu.ccgslb.net/ is not usable
-                # so retry until got a working URL
-                video_url = 'newflv.sohu.ccgslb.net'
-                retries = 0
-                while 'newflv.sohu.ccgslb.net' in video_url and retries < 5:
-                    download_note = 'Download information from CDN gateway for format ' + format_id
-                    if retries > 0:
-                        download_note += ' (retry #%d)' % retries
-                    retries += 1
-                    cdn_info = self._download_json(
-                        'http://data.vod.itc.cn/cdnList?new=' + data['su'][i],
-                        video_id, download_note)
-                    video_url = cdn_info['url']
+                data = format_data['data']
+                clips_url = data['clipsURL']
+                su = data['su']
+
+                part_str = self._download_webpage(
+                    'http://%s/?prot=%s&file=%s&new=%s' %
+                    (allot, prot, clips_url[i], su[i]),
+                    video_id,
+                    'Downloading %s video URL part %d of %d'
+                    % (format_id, i + 1, part_count))
+
+                part_info = part_str.split('|')
+
+                video_url = sanitize_url_path_consecutive_slashes(
+                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({
                     'url': video_url,

From 98ca102441624c2a1a66114c32e0142899f78dd3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Jun 2015 00:59:55 +0800
Subject: [PATCH 0469/2145] [sohu] Fix extraction again

---
 youtube_dl/extractor/sohu.py | 42 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7644cc02d..ba2d5e19b 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,10 +6,10 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_request
+    compat_urllib_request,
+    compat_urllib_parse,
 )
 from ..utils import (
-    sanitize_url_path_consecutive_slashes,
     ExtractorError,
 )
 
@@ -143,23 +143,41 @@ class SohuIE(InfoExtractor):
             formats = []
             for format_id, format_data in formats_json.items():
                 allot = format_data['allot']
-                prot = format_data['prot']
 
                 data = format_data['data']
                 clips_url = data['clipsURL']
                 su = data['su']
 
-                part_str = self._download_webpage(
-                    'http://%s/?prot=%s&file=%s&new=%s' %
-                    (allot, prot, clips_url[i], su[i]),
-                    video_id,
-                    'Downloading %s video URL part %d of %d'
-                    % (format_id, i + 1, part_count))
+                video_url = 'newflv.sohu.ccgslb.net'
+                cdnId = None
+                retries = 0
 
-                part_info = part_str.split('|')
+                while 'newflv.sohu.ccgslb.net' in video_url:
+                    params = {
+                        'prot': 9,
+                        'file': clips_url[i],
+                        'new': su[i],
+                        'prod': 'flash',
+                    }
 
-                video_url = sanitize_url_path_consecutive_slashes(
-                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
+                    if cdnId is not None:
+                        params['idc'] = cdnId
+
+                    download_note = 'Downloading %s video URL part %d of %d' % (
+                        format_id, i + 1, part_count)
+
+                    if retries > 0:
+                        download_note += ' (retry #%d)' % retries
+                    part_info = self._parse_json(self._download_webpage(
+                        'http://%s/?%s' % (allot, compat_urllib_parse.urlencode(params)),
+                        video_id, download_note), video_id)
+
+                    video_url = part_info['url']
+                    cdnId = part_info.get('nid')
+
+                    retries += 1
+                    if retries > 5:
+                        raise ExtractorError('Failed to get video URL')
 
                 formats.append({
                     'url': video_url,

From 0bbba43ed0c68b612fcafbdad460a93b733b9f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:10:38 +0600
Subject: [PATCH 0470/2145] [xhamster:embed] Add extractor (Closes #6032)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/xhamster.py | 28 +++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd3c3193f..dc1a302e6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -701,7 +701,10 @@ from .wrzuta import WrzutaIE
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
-from .xhamster import XHamsterIE
+from .xhamster import (
+    XHamsterIE,
+    XHamsterEmbedIE,
+)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 4527567f8..ae3c5962b 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -13,7 +13,6 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    """Information Extractor for xHamster"""
     _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
     _TESTS = [
         {
@@ -133,3 +132,30 @@ class XHamsterIE(InfoExtractor):
             'age_limit': age_limit,
             'formats': formats,
         }
+
+
+class XHamsterEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://xhamster.com/xembed.php?video=3328539',
+        'info_dict': {
+            'id': '3328539',
+            'ext': 'mp4',
+            'title': 'Pen Masturbation',
+            'upload_date': '20140728',
+            'uploader_id': 'anonymous',
+            'duration': 5,
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
+            webpage, 'xhamster url')
+
+        return self.url_result(video_url, 'XHamster');

From 2bb5b6d0a1671957c7a2e6d6433901b2a1b8f48f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:11:25 +0600
Subject: [PATCH 0471/2145] [generic] Add support for xhamster embeds

---
 youtube_dl/extractor/generic.py  | 6 ++++++
 youtube_dl/extractor/xhamster.py | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 07939b196..e108bde66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -43,6 +43,7 @@ from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
+from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 
@@ -1331,6 +1332,11 @@ class GenericIE(InfoExtractor):
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
 
+        # Look for embedded XHamster player
+        xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
+        if xhamster_urls:
+            return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ae3c5962b..725e01ac3 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -149,6 +149,12 @@ class XHamsterEmbedIE(InfoExtractor):
         }
     }
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?xhamster\.com/xembed\.php\?video=\d+)\1',
+            webpage)]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From c76799c5553b3b48bb7cc73dec452c3637a8670a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:18:28 +0600
Subject: [PATCH 0472/2145] [extractor/generic] Add test for xhamster embed

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e108bde66..20fcd8170 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -336,6 +336,15 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # XHamster embed
+        {
+            'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
+            'info_dict': {
+                'id': 'showthread',
+                'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
+            },
+            'playlist_mincount': 7,
+        },
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',

From a5158f38a31e863a39de8f66c26469a5d4469280 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Jun 2015 15:02:53 +0800
Subject: [PATCH 0473/2145] [generic/adobetv] Support AdobeTVVideo embeds
 (#6039)

---
 youtube_dl/extractor/adobetv.py |  1 +
 youtube_dl/extractor/generic.py | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 695a4a15c..5e43adc51 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -77,6 +77,7 @@ class AdobeTVVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
 
     _TEST = {
+        # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
         'url': 'https://video.tv.adobe.com/v/2456/',
         'md5': '43662b577c018ad707a63766462b1e87',
         'info_dict': {
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 20fcd8170..5c03fddc6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -835,6 +835,18 @@ class GenericIE(InfoExtractor):
                 'title': 'Le débat',
                 'thumbnail': 're:^https?://.*\.jpe?g$',
             }
+        },
+        # AdobeTVVideo embed
+        {
+            'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
+            'md5': '43662b577c018ad707a63766462b1e87',
+            'info_dict': {
+                'id': '2456',
+                'ext': 'mp4',
+                'title': 'New experience with Acrobat DC',
+                'description': 'New experience with Acrobat DC',
+                'duration': 248.667,
+            },
         }
     ]
 
@@ -1518,6 +1530,15 @@ class GenericIE(InfoExtractor):
         if dmcloud_url:
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
+        # Look for AdobeTVVideo embeds
+        mobj = re.search(
+            r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                self._proto_relative_url(unescapeHTML(mobj.group(1))),
+                'AdobeTVVideo')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 25701d5a2ca8c7a58c91e11c6a30d4e61b02e89c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 22 Jun 2015 11:18:52 +0200
Subject: [PATCH 0474/2145] [xhamster] pep8: remove trailing ';'

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 725e01ac3..b4ad513a0 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -164,4 +164,4 @@ class XHamsterEmbedIE(InfoExtractor):
             r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
             webpage, 'xhamster url')
 
-        return self.url_result(video_url, 'XHamster');
+        return self.url_result(video_url, 'XHamster')

From 255f5694aabe07fa7f33978c6b97ced469e172db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Jun 2015 20:11:15 +0600
Subject: [PATCH 0475/2145] [faz] Extend _VALID_URL (Closes #6050)

---
 youtube_dl/extractor/faz.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 3c39ca451..cebdd0193 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 
 class FazIE(InfoExtractor):
     IE_NAME = 'faz.net'
-    _VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?faz\.net/(?:[^/]+/)*.*?-(?P<id>\d+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
         'info_dict': {
             'id': '12610585',
@@ -16,7 +16,22 @@ class FazIE(InfoExtractor):
             'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
             'description': 'md5:1453fbf9a0d041d985a47306192ea253',
         },
-    }
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/foobarblafasel-13659345.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From a9dcf4a860214e37971ab05f27f74bbae65ff8ae Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 23 Jun 2015 01:08:07 -0500
Subject: [PATCH 0476/2145] Prefer externalId over non-mediaserver-specific
 hashkey for video id.

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 310db9d1d..fed344ea0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -540,7 +540,7 @@ class BBCNewsIE(BBCCoUkIE):
             self._sort_formats(formats)
 
             ret.append( {
-                'id': jent.get('programme_id',jent.get('id')),
+                'id': jent.get('id') if programme_id == None else programme_id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From e20d0c1e69f66a82dd493680351538ea92393fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Jun 2015 21:34:29 +0600
Subject: [PATCH 0477/2145] [brightcove] Use `compat_xml_parse_error` (Closes
 #6060)

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d768f99e6..4721c2293 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_xml_parse_error,
 )
 from ..utils import (
     determine_ext,
@@ -119,7 +120,7 @@ class BrightcoveIE(InfoExtractor):
 
         try:
             object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
-        except xml.etree.ElementTree.ParseError:
+        except compat_xml_parse_error:
             return
 
         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')

From 2fece970b80022574a6b54c936820897cfd10d70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 03:08:24 +0600
Subject: [PATCH 0478/2145] [extractor/generic] Clarify generic extraction
 warning

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3d672197c..c8582bda9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -977,10 +977,10 @@ class GenericIE(InfoExtractor):
                 'upload_date': upload_date,
             }
 
-        if (not self._downloader.params.get('test', False) and
-                not is_intentional and
-                not self._downloader.params.get('force_generic_extractor', False)):
-            self._downloader.report_warning('Falling back on generic information extractor.')
+        if not self._downloader.params.get('test', False) and not is_intentional:
+            force = self._downloader.params.get('force_generic_extractor', False)
+            self._downloader.report_warning(
+                '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
         if not full_response:
             request = compat_urllib_request.Request(url)

From 18b5e1e5348ba3a6d1b6a98e97217eebb3d32a1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Jun 2015 16:00:12 +0800
Subject: [PATCH 0479/2145] [drbonanza] Fix extraction of videos

---
 youtube_dl/extractor/drbonanza.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 7626219ba..8b98b013a 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -15,7 +15,6 @@ class DRBonanzaIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
-        'md5': 'fe330252ddea607635cf2eb2c99a0af3',
         'info_dict': {
             'id': '65517',
             'ext': 'mp4',
@@ -26,6 +25,9 @@ class DRBonanzaIE(InfoExtractor):
             'upload_date': '20110120',
             'duration': 3664,
         },
+        'params': {
+            'skip_download': True,  # requires rtmp
+        },
     }, {
         'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
         'md5': '6dfe039417e76795fb783c52da3de11d',
@@ -93,6 +95,11 @@ class DRBonanzaIE(InfoExtractor):
                         'format_id': file['Type'].replace('Video', ''),
                         'preference': preferencemap.get(file['Type'], -10),
                     })
+                    if format['url'].startswith('rtmp'):
+                        rtmp_url = format['url']
+                        format['rtmp_live'] = True  # --resume does not work
+                        if '/bonanza/' in rtmp_url:
+                            format['play_path'] = rtmp_url.split('/bonanza/')[1]
                     formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
@@ -111,9 +118,6 @@ class DRBonanzaIE(InfoExtractor):
         description = '%s\n%s\n%s\n' % (
             info['Description'], info['Actors'], info['Colophon'])
 
-        for f in formats:
-            f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
-            f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
         self._sort_formats(formats)
 
         display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id

From 1a1251e87716ba02bfe6c4c9af2726e413e00208 Mon Sep 17 00:00:00 2001
From: Mitsukarenai <mitsu@suumitsu.eu>
Date: Wed, 24 Jun 2015 10:40:01 +0200
Subject: [PATCH 0480/2145] [noco.tv] Fix issue #6066: title interpreted as
 integer

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 5bbd2dcf6..a53e27b27 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -195,7 +195,7 @@ class NocoIE(InfoExtractor):
         if episode_number:
             title += ' #' + compat_str(episode_number)
         if episode:
-            title += ' - ' + episode
+            title += ' - ' + compat_str(episode)
 
         description = show.get('show_resume') or show.get('family_resume')
 

From a219d175c66680968d5ce7de0a2e619623171a40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 20:48:57 +0600
Subject: [PATCH 0481/2145] [soundcloud] Improve _VALID_URL

Do not takeover soundcloud:user URLs
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c23c5ee0f..e31b4596a 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!sets/|likes/?(?:$|[?#]))
+                            (?!sets/|(?:likes|tracks)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)

From 03b9c94437756c3f715b29150daa59a8ea97eebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 20:49:22 +0600
Subject: [PATCH 0482/2145] [soundcloud:user] Add test

---
 youtube_dl/extractor/soundcloud.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e31b4596a..118ca4832 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -307,6 +307,9 @@ class SoundcloudUserIE(SoundcloudIE):
             'title': 'The Royal Concept',
         },
         'playlist_mincount': 1,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f843300fe56ffbfc8e3005fd0f7a8237e5deaaae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:12:13 +0600
Subject: [PATCH 0483/2145] [onionstudios] Add extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/onionstudios.py | 67 ++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/onionstudios.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc1a302e6..46cc4cd06 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -388,6 +388,7 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
+from .onionstudios import OnionStudiosIE
 from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
new file mode 100644
index 000000000..d5d03fd44
--- /dev/null
+++ b/youtube_dl/extractor/onionstudios.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import determine_ext
+
+
+class OnionStudiosIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:videos/[^/]+-|embed\?.*\bid=)(?P<id>\d+)(?!-)'
+
+    _TESTS = [{
+        'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
+        'md5': 'd4851405d31adfadf71cd7a487b765bb',
+        'info_dict': {
+            'id': '2937',
+            'ext': 'mp4',
+            'title': 'Hannibal charges forward, stops for a cocktail',
+            'description': 'md5:545299bda6abf87e5ec666548c6a9448',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'The A.V. Club',
+            'uploader_id': 'TheAVClub',
+        },
+    }, {
+        'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://www.onionstudios.com/embed?id=%s' % video_id, video_id)
+
+        formats = []
+        for src in re.findall(r'<source[^>]+src="([^"]+)"', webpage):
+            if determine_ext(src) != 'm3u8':  # m3u8 always results in 403
+                formats.append({
+                    'url': src,
+                })
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'share_title\s*=\s*"([^"]+)"', webpage, 'title')
+        description = self._search_regex(
+            r'share_description\s*=\s*"([^"]+)"', webpage,
+            'description', default=None)
+        thumbnail = self._search_regex(
+            r'poster="([^"]+)"', webpage, 'thumbnail', default=False)
+
+        uploader_id = self._search_regex(
+            r'twitter_handle\s*=\s*"([^"]+)"',
+            webpage, 'uploader id', fatal=False)
+        uploader = self._search_regex(
+            r'window\.channelName\s*=\s*"Embedded:([^"]+)"',
+            webpage, 'uploader', default=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
+        }

From d4f58034f7f3d959788d4fcf3812e24d3ba05f68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:16:33 +0600
Subject: [PATCH 0484/2145] [onionstudios] Add generic embed extraction routine

---
 youtube_dl/extractor/onionstudios.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index d5d03fd44..8fa507dec 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -27,6 +27,13 @@ class OnionStudiosIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/embed.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 1ac1c4c26ecdccb8c515aac634d965b89cbdabe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:19:50 +0600
Subject: [PATCH 0485/2145] [extractor/generic] Add support for OnionStudios
 embeds (Closes #5841)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7769ffc5c..9c5125f38 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -46,6 +46,7 @@ from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
+from .onionstudios import OnionStudiosIE
 
 
 class GenericIE(InfoExtractor):
@@ -1532,6 +1533,11 @@ class GenericIE(InfoExtractor):
         if dmcloud_url:
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
+        # Look for OnionStudios embeds
+        onionstudios_url = OnionStudiosIE._extract_url(webpage)
+        if onionstudios_url:
+            return self.url_result(onionstudios_url)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 8084be78c50e0e921508f13e69fda34a907dc5f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:23:16 +0600
Subject: [PATCH 0486/2145] [extractor/generic] Add test for OnionStudios
 embeds

---
 youtube_dl/extractor/generic.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9c5125f38..42e4e7035 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -837,6 +837,18 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpe?g$',
             }
         },
+        # OnionStudios embed
+        {
+            'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
+            'info_dict': {
+                'id': '2855',
+                'ext': 'mp4',
+                'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
+                'thumbnail': 're:^https?://.*\.jpe?g$',
+                'uploader': 'ClickHole',
+                'uploader_id': 'clickhole',
+            }
+        },
         # AdobeTVVideo embed
         {
             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',

From 12e9e8445dc090a12c6f7643a346725788b0ca6b Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Thu, 25 Jun 2015 01:10:42 +0600
Subject: [PATCH 0487/2145] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5f3a08f5a..10d2d4209 100644
--- a/README.md
+++ b/README.md
@@ -379,7 +379,7 @@ In February 2015, the new YouTube player contained a character sequence in a str
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
-These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--network-address` options](#network-options) to select another IP address.
+These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--source-address` options](#network-options) to select another IP address.
 
 ### SyntaxError: Non-ASCII character ###
 

From da92eeae42f556926cb676b3c14e270603b7e38e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Thu, 25 Jun 2015 00:31:32 -0500
Subject: [PATCH 0488/2145] Fix tests, description formatting

---
 youtube_dl/extractor/bbc.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index fed344ea0..bb671d473 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -428,6 +428,8 @@ class BBCNewsIE(BBCCoUkIE):
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
+            'upload_date': '20150324',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -438,8 +440,11 @@ class BBCNewsIE(BBCCoUkIE):
         'info_dict': {
             'id': 'NA',
             'ext': 'mp4',
-            'title': 'YPG - Tel Abyad..n tamam. kontrol.m.zde',
+            'title': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
+            'description': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
             'duration': 47,
+            'upload_date': '20150615',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -450,8 +455,11 @@ class BBCNewsIE(BBCCoUkIE):
         'info_dict': {
             'id': '39275083',
             'ext': 'mp4',
-            'title': 'Honduras militariza sus hospitales por nuevo esc.ndalo de corrupci.n',
+            'title': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
+            'description': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
             'duration': 87,
+            'upload_date': '20150619',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -507,7 +515,9 @@ class BBCNewsIE(BBCCoUkIE):
             title = jent.get('caption',list_title)
 
             duration = parse_duration(jent.get('duration'))
-            description = list_title + ' - ' + jent.get('caption','')
+            description = list_title
+            if jent.get('caption'):
+               description += ' - ' + jent.get('caption')
             thumbnail = None
             if jent.has_key('image'):
                thumbnail=jent['image'].get('href')
@@ -539,8 +549,12 @@ class BBCNewsIE(BBCCoUkIE):
                
             self._sort_formats(formats)
 
+            id = jent.get('id') if programme_id == None else programme_id
+            if id == None:
+               id = 'NA'
+
             ret.append( {
-                'id': jent.get('id') if programme_id == None else programme_id,
+                'id': id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From a1593a4a0e0f278d15ebda603df8c69679bb34c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Jun 2015 07:43:49 +0200
Subject: [PATCH 0489/2145] release 2015.06.25

---
 README.md              | 1 +
 docs/supportedsites.md | 7 ++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 10d2d4209..e3452c9e1 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ which means you can modify it, redistribute it or use it however you like.
     --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors
     --extractor-descriptions         Output descriptions of all supported extractors
+    --force-generic-extractor        Force extraction to use the generic extractor
     --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
                                      default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 220e52b98..9a50fbd1c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -17,6 +17,7 @@
  - **AcademicEarth:Course**
  - **AddAnime**
  - **AdobeTV**
+ - **AdobeTVVideo**
  - **AdultSwim**
  - **Aftenposten**
  - **Aftonbladet**
@@ -110,6 +111,7 @@
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
+ - **DailymotionCloud**
  - **daum.net**
  - **DBTV**
  - **DctpTv**
@@ -344,6 +346,7 @@
  - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
+ - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
  - **OpenFilm**
@@ -357,6 +360,7 @@
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
+ - **Pinkbike**
  - **Pladform**
  - **PlanetaPlay**
  - **play.fm**
@@ -612,6 +616,7 @@
  - **XBef**
  - **XboxClips**
  - **XHamster**
+ - **XHamsterEmbed**
  - **XMinus**
  - **XNXX**
  - **Xstream**
@@ -628,7 +633,7 @@
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
- - **Youku**
+ - **youku**
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34a13cb81..a225e03a1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.15'
+__version__ = '2015.06.25'

From 2182ab5187736f806f1a2ce959c3893179bdd1a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:15:13 +0800
Subject: [PATCH 0490/2145] [thesixtyone] Fix audio_server

Some of the songs are moved to Amazon AWS
---
 youtube_dl/extractor/thesixtyone.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index a77c6a2fc..e1be978a4 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -17,7 +17,7 @@ class TheSixtyOneIE(InfoExtractor):
             song
         )/(?P<id>[A-Za-z0-9]+)/?$'''
     _SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
-    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}.thesixtyone.com/thesixtyone_production/audio/{0:}_stream'
+    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}/thesixtyone_production/audio/{0:}_stream'
     _THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
     _TESTS = [
         {
@@ -78,6 +78,12 @@ class TheSixtyOneIE(InfoExtractor):
 
         song_data = json.loads(self._search_regex(
             r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+
+        if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
+            song_data['audio_server'] = 's3.amazonaws.com'
+        else:
+            song_data['audio_server'] = song_data['audio_server'] + '.thesixtyone.com'
+
         keys = [self._DECODE_MAP.get(s, s) for s in song_data['key']]
         url = self._SONG_FILE_URL_TEMPLATE.format(
             "".join(reversed(keys)), **song_data)

From 381c06775598da9667548fcc45d5df7a03748f7d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:19:04 +0800
Subject: [PATCH 0491/2145] [thesixtyone] Modernize

---
 youtube_dl/extractor/thesixtyone.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index e1be978a4..5d09eb9a8 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -1,9 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
@@ -70,14 +67,13 @@ class TheSixtyOneIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        song_id = mobj.group('id')
+        song_id = self._match_id(url)
 
         webpage = self._download_webpage(
             self._SONG_URL_TEMPLATE.format(song_id), song_id)
 
-        song_data = json.loads(self._search_regex(
-            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+        song_data = self._parse_json(self._search_regex(
+            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'), song_id)
 
         if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
             song_data['audio_server'] = 's3.amazonaws.com'

From b8091db6b9c75fbdc0ccaad3a9503ef091a7437c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:40:29 +0800
Subject: [PATCH 0492/2145] [planetaplay/quickvid/vube] Skip inaccessible sites

---
 youtube_dl/extractor/planetaplay.py | 3 ++-
 youtube_dl/extractor/quickvid.py    | 1 +
 youtube_dl/extractor/vube.py        | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/planetaplay.py b/youtube_dl/extractor/planetaplay.py
index 596c621d7..06505e96f 100644
--- a/youtube_dl/extractor/planetaplay.py
+++ b/youtube_dl/extractor/planetaplay.py
@@ -18,7 +18,8 @@ class PlanetaPlayIE(InfoExtractor):
             'id': '3586',
             'ext': 'flv',
             'title': 'md5:e829428ee28b1deed00de90de49d1da1',
-        }
+        },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     _SONG_FORMATS = {
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
index af7d76cf4..f414e2384 100644
--- a/youtube_dl/extractor/quickvid.py
+++ b/youtube_dl/extractor/quickvid.py
@@ -24,6 +24,7 @@ class QuickVidIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
             'view_count': int,
         },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 405cb9db4..149e36467 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -36,6 +36,7 @@ class VubeIE(InfoExtractor):
                 'comment_count': int,
                 'categories': ['amazing', 'hd', 'best drummer ever', 'william wei', 'bucket drumming', 'street drummer', 'epic street drumming'],
             },
+            'skip': 'Not accessible from Travis CI server',
         }, {
             'url': 'http://vube.com/Chiara+Grispo+Video+Channel/YL2qNPkqon',
             'md5': 'db7aba89d4603dadd627e9d1973946fe',

From 906e2f0eac3729228644fc5e9a3e8dedbd74894e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 25 Jun 2015 15:48:04 +0200
Subject: [PATCH 0493/2145] [downloader/external] Add downloader for httpie
 (closes #6079)

---
 youtube_dl/downloader/external.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7ca2d3143..a57c15856 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -109,6 +109,14 @@ class Aria2cFD(ExternalFD):
         cmd += ['--', info_dict['url']]
         return cmd
 
+
+class HttpieFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
+        for key, val in info_dict['http_headers'].items():
+            cmd += ['%s:%s' % (key, val)]
+        return cmd
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()

From 533f67d3fa76eb44799276656207a25ece6d20a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Jun 2015 19:54:44 +0600
Subject: [PATCH 0494/2145] [infoq] Relax _VALID_URL (Closes #6071)

---
 youtube_dl/extractor/infoq.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index f25f43664..117a7faf6 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -9,9 +9,9 @@ from ..compat import (
 
 
 class InfoQIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?infoq\.com/[^/]+/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?infoq\.com/(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things',
         'md5': 'b5ca0e0a8c1fed93b0e65e48e462f9a2',
         'info_dict': {
@@ -20,7 +20,10 @@ class InfoQIE(InfoExtractor):
             'description': 'Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.',
             'title': 'A Few of My Favorite [Python] Things',
         },
-    }
+    }, {
+        'url': 'http://www.infoq.com/fr/presentations/changez-avis-sur-javascript',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From b4dea075a3f923b4d85d74729479ec4c3e3a313a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:36:04 +0600
Subject: [PATCH 0495/2145] [lynda] Fix login request (Closes #6088)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index cfd3b14f4..2ff902c65 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -36,7 +36,7 @@ class LyndaBaseIE(InfoExtractor):
             'stayPut': 'false'
         }
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From 62cca96b729424fa71a3b174af7b646cc1cf12f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:46:42 +0600
Subject: [PATCH 0496/2145] [lynda] Fix confirm login request (#6088)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 2ff902c65..18efe5a09 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -65,7 +65,7 @@ class LyndaBaseIE(InfoExtractor):
                     'stayPut': 'false',
                 }
                 request = compat_urllib_request.Request(
-                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
+                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
                 login_page = self._download_webpage(
                     request, None,
                     'Confirming log in and log out from another device')

From 2988835af5cbe695e61d3b58bc65027f5b48fb63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:48:23 +0600
Subject: [PATCH 0497/2145] [lynda] Fix non-ASCII logins/passwords on python 2

---
 youtube_dl/extractor/lynda.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 18efe5a09..a00f6e5e5 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -30,8 +30,8 @@ class LyndaBaseIE(InfoExtractor):
             return
 
         login_form = {
-            'username': username,
-            'password': password,
+            'username': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
             'remember': 'false',
             'stayPut': 'false'
         }

From 2028c6e03d7e254831350081bb4b4741b0b47ac4 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Fri, 26 Jun 2015 21:27:43 +0530
Subject: [PATCH 0498/2145] Added a Playlist Info Extractor for WebOfStories

---
 youtube_dl/extractor/__init__.py     |  5 ++++-
 youtube_dl/extractor/webofstories.py | 25 +++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 46cc4cd06..c3f3a3e38 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -693,7 +693,10 @@ from .wdr import (
     WDRMobileIE,
     WDRMausIE,
 )
-from .webofstories import WebOfStoriesIE
+from .webofstories import (
+    WebOfStoriesIE,
+    WebOfStoriesPlaylistIE,
+)
 from .weibo import WeiboIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 73077a312..d70e30c00 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -98,3 +100,26 @@ class WebOfStoriesIE(InfoExtractor):
             'description': description,
             'duration': duration,
         }
+
+
+class WebOfStoriesPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
+    _TESTS = []
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.webofstories.com/play/%s' % video_number, 'WebOfStories')
+            for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
+        ]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Web\sof\sStories</title>', webpage, 'title')
+
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 03339b7b5bb8f563eace8826512e2f7a4baba415 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 18:25:43 +0100
Subject: [PATCH 0499/2145] [snagfilms] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/snagfilms.py | 67 +++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/snagfilms.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc1a302e6..3b906b880 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -492,6 +492,7 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
+from .snagfilms import SnagFilmsIE
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soompi import (
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
new file mode 100644
index 000000000..33832855f
--- /dev/null
+++ b/youtube_dl/extractor/snagfilms.py
@@ -0,0 +1,67 @@
+from .common import InfoExtractor
+from ..utils import js_to_json
+from re import DOTALL
+
+class SnagFilmsIE(InfoExtractor):
+	_VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
+	_TEST = {
+		'url': 'http://www.snagfilms.com/films/title/lost_for_life',
+		'info_dict':
+		{
+			'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
+			'display_id': 'lost_for_life',
+			'ext': 'mp4',
+			'title': 'Lost for Life',
+			'duration': 4489,
+			'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
+			'categories': ['Documentary','Crime','Award Winning','Festivals']
+		}
+	}
+
+	def _real_extract(self, url):
+		display_id = self._search_regex(
+			self._VALID_URL,
+			url,
+			'display_id',
+			group='display_id'
+		)
+		webpage = self._download_webpage(url, display_id)
+
+		json_data = self._parse_json(self._html_search_regex(
+			r'"data":{"film":(?P<data>{.*?}})}',
+			webpage,
+			'data',
+			group='data'
+		), display_id)
+		title = json_data['title']
+		video_id = json_data['id']
+		duration = int(json_data['duration'])
+		description = json_data['synopsis']
+		categories = [category['title'] for category in json_data['categories']]
+
+		embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+		sources = self._parse_json(js_to_json(self._html_search_regex(
+			r'sources: (?P<sources>\[.*?\])',
+			embed_webpage,
+			'sources',
+			group='sources',
+			flags=DOTALL
+		)), video_id)
+
+		formats = []
+		for source in sources:
+			if source['type'] == 'm3u8':
+				formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+			else:
+				formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
+		self._sort_formats(formats)
+
+		return {
+			'id': video_id,
+			'display_id': display_id,
+			'title': title,
+			'duration': duration,
+			'description': description,
+			'categories': categories,
+			'formats': formats,
+		}

From d8d24a922adb0646f9f78b2fc4a287a72fa7ff73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 00:36:23 +0600
Subject: [PATCH 0500/2145] [youtube] Extract formats from multiple DASH
 manifests (Closes #6093)

DASH manifest pointed by dashmpd from the video webpage and one pointed by get_video_info may
be different (namely different itag set) - some itags are missing from DASH manifest pointed by
webpage's dashmpd, some - from DASH manifest pointed by get_video_info's dashmpd).
The general idea is to take a union of itags of both DASH manifests (for example video with such
'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093).
---
 youtube_dl/extractor/youtube.py | 77 ++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..fa1a2e544 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -853,6 +853,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             player_url = None
 
+        dash_mpds = []
+
+        def add_dash_mpd(video_info):
+            dash_mpd = video_info.get('dashmpd')
+            if dash_mpd and dash_mpd[0] not in dash_mpds:
+                dash_mpds.append(dash_mpd[0])
+
         # Get video info
         embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
@@ -873,34 +880,40 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 note='Refetching age-gated info webpage',
                 errnote='unable to download video info webpage')
             video_info = compat_parse_qs(video_info_webpage)
+            add_dash_mpd(video_info)
         else:
             age_gate = False
-            try:
-                # Try looking directly into the video webpage
-                mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
-                if not mobj:
-                    raise ValueError('Could not find ytplayer.config')  # caught below
+            # Try looking directly into the video webpage
+            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+            if mobj:
                 json_code = uppercase_escape(mobj.group(1))
                 ytplayer_config = json.loads(json_code)
                 args = ytplayer_config['args']
-                # Convert to the same format returned by compat_parse_qs
-                video_info = dict((k, [v]) for k, v in args.items())
-                if not args.get('url_encoded_fmt_stream_map'):
-                    raise ValueError('No stream_map present')  # caught below
-            except ValueError:
-                # We fallback to the get_video_info pages (used by the embed page)
-                self.report_video_info_webpage_download(video_id)
-                for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (proto, video_id, el_type))
-                    video_info_webpage = self._download_webpage(
-                        video_info_url,
-                        video_id, note=False,
-                        errnote='unable to download video info webpage')
-                    video_info = compat_parse_qs(video_info_webpage)
-                    if 'token' in video_info:
-                        break
+                if args.get('url_encoded_fmt_stream_map'):
+                    # Convert to the same format returned by compat_parse_qs
+                    video_info = dict((k, [v]) for k, v in args.items())
+                    add_dash_mpd(video_info)
+            # We also try looking in get_video_info since it may contain different dashmpd
+            # URL that points to a DASH manifest with possibly different itag set (some itags
+            # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
+            # manifest pointed by get_video_info's dashmpd).
+            # The general idea is to take a union of itags of both DASH manifests (for example
+            # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+            self.report_video_info_webpage_download(video_id)
+            for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                video_info_url = (
+                    '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                    % (proto, video_id, el_type))
+                video_info_webpage = self._download_webpage(
+                    video_info_url,
+                    video_id, note=False,
+                    errnote='unable to download video info webpage')
+                get_video_info = compat_parse_qs(video_info_webpage)
+                add_dash_mpd(get_video_info)
+                if not video_info:
+                    video_info = get_video_info
+                if 'token' in get_video_info:
+                    break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(
@@ -1118,24 +1131,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
-            dash_mpd = video_info.get('dashmpd')
-            if dash_mpd:
-                dash_manifest_url = dash_mpd[0]
+            for dash_manifest_url in dash_mpds:
+                dash_formats = {}
                 try:
-                    dash_formats = self._parse_dash_manifest(
-                        video_id, dash_manifest_url, player_url, age_gate)
+                    for df in self._parse_dash_manifest(
+                            video_id, dash_manifest_url, player_url, age_gate):
+                        # Do not overwrite DASH format found in some previous DASH manifest
+                        if df['format_id'] not in dash_formats:
+                            dash_formats[df['format_id']] = df
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
-                else:
+                if dash_formats:
                     # Remove the formats we found through non-DASH, they
                     # contain less info and it can be wrong, because we use
                     # fixed values (for example the resolution). See
                     # https://github.com/rg3/youtube-dl/issues/5774 for an
                     # example.
-                    dash_keys = set(df['format_id'] for df in dash_formats)
+                    dash_keys = set(df['format_id'] for df in dash_formats.values())
                     formats = [f for f in formats if f['format_id'] not in dash_keys]
-                    formats.extend(dash_formats)
+                    formats.extend(dash_formats.values())
 
         # Check for malformed aspect ratio
         stretched_m = re.search(

From 1b5a1ae257c6d64fd69ea91eaf69328d4b6d17df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 00:41:26 +0600
Subject: [PATCH 0501/2145] [youtube] Pick up codecs info from DASH manifest
 when not set explicitly

---
 youtube_dl/extractor/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..c9d8e5125 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -824,6 +824,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     except StopIteration:
                         full_info = self._formats.get(format_id, {}).copy()
                         full_info.update(f)
+                        codecs = r.attrib.get('codecs')
+                        if codecs:
+                            if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
+                                full_info['vcodec'] = codecs
+                            elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
+                                full_info['acodec'] = codecs
                         formats.append(full_info)
                     else:
                         existing_format.update(f)

From d80265ccd6ff08ef273d83aff847a457f051071f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 02:48:50 +0600
Subject: [PATCH 0502/2145] [youtube] Simplify non-DASH formats exclusion

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fa1a2e544..46841617a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1148,8 +1148,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # fixed values (for example the resolution). See
                     # https://github.com/rg3/youtube-dl/issues/5774 for an
                     # example.
-                    dash_keys = set(df['format_id'] for df in dash_formats.values())
-                    formats = [f for f in formats if f['format_id'] not in dash_keys]
+                    formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
                     formats.extend(dash_formats.values())
 
         # Check for malformed aspect ratio

From 7e0480ae0e7a9d0b3c73366b8d9ecf86226ad5a8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 21:50:27 +0100
Subject: [PATCH 0503/2145] convert tabs to 4 spaces identation

---
 youtube_dl/extractor/snagfilms.py | 114 +++++++++++++++---------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 33832855f..212fefef7 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -3,65 +3,65 @@ from ..utils import js_to_json
 from re import DOTALL
 
 class SnagFilmsIE(InfoExtractor):
-	_VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
-	_TEST = {
-		'url': 'http://www.snagfilms.com/films/title/lost_for_life',
-		'info_dict':
-		{
-			'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
-			'display_id': 'lost_for_life',
-			'ext': 'mp4',
-			'title': 'Lost for Life',
-			'duration': 4489,
-			'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
-			'categories': ['Documentary','Crime','Award Winning','Festivals']
-		}
-	}
+    _VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
+    _TEST = {
+        'url': 'http://www.snagfilms.com/films/title/lost_for_life',
+        'info_dict':
+        {
+            'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
+            'display_id': 'lost_for_life',
+            'ext': 'mp4',
+            'title': 'Lost for Life',
+            'duration': 4489,
+            'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
+            'categories': ['Documentary','Crime','Award Winning','Festivals']
+        }
+    }
 
-	def _real_extract(self, url):
-		display_id = self._search_regex(
-			self._VALID_URL,
-			url,
-			'display_id',
-			group='display_id'
-		)
-		webpage = self._download_webpage(url, display_id)
+    def _real_extract(self, url):
+        display_id = self._search_regex(
+            self._VALID_URL,
+            url,
+            'display_id',
+            group='display_id'
+        )
+        webpage = self._download_webpage(url, display_id)
 
-		json_data = self._parse_json(self._html_search_regex(
-			r'"data":{"film":(?P<data>{.*?}})}',
-			webpage,
-			'data',
-			group='data'
-		), display_id)
-		title = json_data['title']
-		video_id = json_data['id']
-		duration = int(json_data['duration'])
-		description = json_data['synopsis']
-		categories = [category['title'] for category in json_data['categories']]
+        json_data = self._parse_json(self._html_search_regex(
+            r'"data":{"film":(?P<data>{.*?}})}',
+            webpage,
+            'data',
+            group='data'
+        ), display_id)
+        title = json_data['title']
+        video_id = json_data['id']
+        duration = int(json_data['duration'])
+        description = json_data['synopsis']
+        categories = [category['title'] for category in json_data['categories']]
 
-		embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-		sources = self._parse_json(js_to_json(self._html_search_regex(
-			r'sources: (?P<sources>\[.*?\])',
-			embed_webpage,
-			'sources',
-			group='sources',
-			flags=DOTALL
-		)), video_id)
+        embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+        sources = self._parse_json(js_to_json(self._html_search_regex(
+            r'sources: (?P<sources>\[.*?\])',
+            embed_webpage,
+            'sources',
+            group='sources',
+            flags=DOTALL
+        )), video_id)
 
-		formats = []
-		for source in sources:
-			if source['type'] == 'm3u8':
-				formats.extend(self._extract_m3u8_formats(source['file'], video_id))
-			else:
-				formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
-		self._sort_formats(formats)
+        formats = []
+        for source in sources:
+            if source['type'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+            else:
+                formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
+        self._sort_formats(formats)
 
-		return {
-			'id': video_id,
-			'display_id': display_id,
-			'title': title,
-			'duration': duration,
-			'description': description,
-			'categories': categories,
-			'formats': formats,
-		}
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'duration': duration,
+            'description': description,
+            'categories': categories,
+            'formats': formats,
+        }

From fd40bdc0be8984bf6043e70796c1d465a0499d03 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 21:56:15 +0100
Subject: [PATCH 0504/2145] remove unnecessary symbolic name for group

---
 youtube_dl/extractor/snagfilms.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 212fefef7..60354b9af 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -31,7 +31,6 @@ class SnagFilmsIE(InfoExtractor):
             r'"data":{"film":(?P<data>{.*?}})}',
             webpage,
             'data',
-            group='data'
         ), display_id)
         title = json_data['title']
         video_id = json_data['id']
@@ -44,7 +43,6 @@ class SnagFilmsIE(InfoExtractor):
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,
             'sources',
-            group='sources',
             flags=DOTALL
         )), video_id)
 

From 7d7d4690259f343385a240efcef8d157fc99c72d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 27 Jun 2015 00:13:14 +0100
Subject: [PATCH 0505/2145] add support for embed links

---
 youtube_dl/extractor/snagfilms.py | 48 ++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 60354b9af..cd345474e 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -1,10 +1,10 @@
+from re import match,DOTALL
 from .common import InfoExtractor
 from ..utils import js_to_json
-from re import DOTALL
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
-    _TEST = {
+    _VALID_URL = r'(?:https?://)?(?:www.|embed.)?snagfilms\.com/(?:films/title/(?P<display_id>.+?)|embed/player\?.*filmId=(?P<id>.+?))(?:&|/|$)'
+    _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'info_dict':
         {
@@ -16,29 +16,48 @@ class SnagFilmsIE(InfoExtractor):
             'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
             'categories': ['Documentary','Crime','Award Winning','Festivals']
         }
-    }
+    },{
+        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831',
+        'info_dict':
+        {
+            'id': '74849a00-85a9-11e1-9660-123139220831',
+            'display_id': 'while_we_watch',
+            'ext': 'mp4',
+            'title': '#whilewewatch',
+            'duration': 2311,
+            'description': 'A gripping portrait of the Occupy Wall Street media revolution,&nbsp;#WHILEWEWATCH is the first definitive film to emerge from Zuccotti Park—with full access and cooperation from masterminds who made #OccupyWallStreet a reality.&nbsp;The #OccupyWallStreet media team had no fear of a critical city government, big corporations, hostile police or a lagging mainstream media to tell their story. Through rain, snow, grueling days and sleeping on concrete, they pump out exhilarating ideas to the world. With little money, they rely on Twitter, texting, Wi-Fi, posters, Tumblr, live streams, YouTube, Facebook, dramatic marches, drumbeats and chants. As the film unfolds, we witness the burgeoning power of social media.<br />',
+            'categories': ['Documentary','Politics']
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id = self._search_regex(
-            self._VALID_URL,
-            url,
-            'display_id',
-            group='display_id'
-        )
-        webpage = self._download_webpage(url, display_id)
+        display_id, video_id = match(self._VALID_URL,url).groups()
+        if display_id is None:
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+
+            display_id = self._html_search_regex(
+                r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
+                embed_webpage,
+                'display_id'
+            )
+        webpage = self._download_webpage('http://www.snagfilms.com/films/title/' + display_id, display_id)
 
         json_data = self._parse_json(self._html_search_regex(
             r'"data":{"film":(?P<data>{.*?}})}',
             webpage,
-            'data',
+            'data'
         ), display_id)
+
         title = json_data['title']
-        video_id = json_data['id']
         duration = int(json_data['duration'])
         description = json_data['synopsis']
         categories = [category['title'] for category in json_data['categories']]
+        thumbnail = json_data['image']
+
+        if video_id is None:
+            video_id = json_data['id']
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
 
-        embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
         sources = self._parse_json(js_to_json(self._html_search_regex(
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,
@@ -61,5 +80,6 @@ class SnagFilmsIE(InfoExtractor):
             'duration': duration,
             'description': description,
             'categories': categories,
+            'thumbnail': thumbnail,
             'formats': formats,
         }

From 4eb10f66216b5239a9a8efcf950c9b16f5c8b5de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Jun 2015 13:13:57 +0800
Subject: [PATCH 0506/2145] [utils] Add ISO3166Utils

---
 youtube_dl/utils.py | 260 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 260 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2746b2d1..96490f112 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2084,6 +2084,266 @@ class ISO639Utils(object):
                 return short_name
 
 
+class ISO3166Utils(object):
+    # From http://data.okfn.org/data/core/country-list
+    _country_map = {
+        'AF': 'Afghanistan',
+        'AX': 'Åland Islands',
+        'AL': 'Albania',
+        'DZ': 'Algeria',
+        'AS': 'American Samoa',
+        'AD': 'Andorra',
+        'AO': 'Angola',
+        'AI': 'Anguilla',
+        'AQ': 'Antarctica',
+        'AG': 'Antigua and Barbuda',
+        'AR': 'Argentina',
+        'AM': 'Armenia',
+        'AW': 'Aruba',
+        'AU': 'Australia',
+        'AT': 'Austria',
+        'AZ': 'Azerbaijan',
+        'BS': 'Bahamas',
+        'BH': 'Bahrain',
+        'BD': 'Bangladesh',
+        'BB': 'Barbados',
+        'BY': 'Belarus',
+        'BE': 'Belgium',
+        'BZ': 'Belize',
+        'BJ': 'Benin',
+        'BM': 'Bermuda',
+        'BT': 'Bhutan',
+        'BO': 'Bolivia, Plurinational State of',
+        'BQ': 'Bonaire, Sint Eustatius and Saba',
+        'BA': 'Bosnia and Herzegovina',
+        'BW': 'Botswana',
+        'BV': 'Bouvet Island',
+        'BR': 'Brazil',
+        'IO': 'British Indian Ocean Territory',
+        'BN': 'Brunei Darussalam',
+        'BG': 'Bulgaria',
+        'BF': 'Burkina Faso',
+        'BI': 'Burundi',
+        'KH': 'Cambodia',
+        'CM': 'Cameroon',
+        'CA': 'Canada',
+        'CV': 'Cape Verde',
+        'KY': 'Cayman Islands',
+        'CF': 'Central African Republic',
+        'TD': 'Chad',
+        'CL': 'Chile',
+        'CN': 'China',
+        'CX': 'Christmas Island',
+        'CC': 'Cocos (Keeling) Islands',
+        'CO': 'Colombia',
+        'KM': 'Comoros',
+        'CG': 'Congo',
+        'CD': 'Congo, the Democratic Republic of the',
+        'CK': 'Cook Islands',
+        'CR': 'Costa Rica',
+        'CI': 'Côte d\'Ivoire',
+        'HR': 'Croatia',
+        'CU': 'Cuba',
+        'CW': 'Curaçao',
+        'CY': 'Cyprus',
+        'CZ': 'Czech Republic',
+        'DK': 'Denmark',
+        'DJ': 'Djibouti',
+        'DM': 'Dominica',
+        'DO': 'Dominican Republic',
+        'EC': 'Ecuador',
+        'EG': 'Egypt',
+        'SV': 'El Salvador',
+        'GQ': 'Equatorial Guinea',
+        'ER': 'Eritrea',
+        'EE': 'Estonia',
+        'ET': 'Ethiopia',
+        'FK': 'Falkland Islands (Malvinas)',
+        'FO': 'Faroe Islands',
+        'FJ': 'Fiji',
+        'FI': 'Finland',
+        'FR': 'France',
+        'GF': 'French Guiana',
+        'PF': 'French Polynesia',
+        'TF': 'French Southern Territories',
+        'GA': 'Gabon',
+        'GM': 'Gambia',
+        'GE': 'Georgia',
+        'DE': 'Germany',
+        'GH': 'Ghana',
+        'GI': 'Gibraltar',
+        'GR': 'Greece',
+        'GL': 'Greenland',
+        'GD': 'Grenada',
+        'GP': 'Guadeloupe',
+        'GU': 'Guam',
+        'GT': 'Guatemala',
+        'GG': 'Guernsey',
+        'GN': 'Guinea',
+        'GW': 'Guinea-Bissau',
+        'GY': 'Guyana',
+        'HT': 'Haiti',
+        'HM': 'Heard Island and McDonald Islands',
+        'VA': 'Holy See (Vatican City State)',
+        'HN': 'Honduras',
+        'HK': 'Hong Kong',
+        'HU': 'Hungary',
+        'IS': 'Iceland',
+        'IN': 'India',
+        'ID': 'Indonesia',
+        'IR': 'Iran, Islamic Republic of',
+        'IQ': 'Iraq',
+        'IE': 'Ireland',
+        'IM': 'Isle of Man',
+        'IL': 'Israel',
+        'IT': 'Italy',
+        'JM': 'Jamaica',
+        'JP': 'Japan',
+        'JE': 'Jersey',
+        'JO': 'Jordan',
+        'KZ': 'Kazakhstan',
+        'KE': 'Kenya',
+        'KI': 'Kiribati',
+        'KP': 'Korea, Democratic People\'s Republic of',
+        'KR': 'Korea, Republic of',
+        'KW': 'Kuwait',
+        'KG': 'Kyrgyzstan',
+        'LA': 'Lao People\'s Democratic Republic',
+        'LV': 'Latvia',
+        'LB': 'Lebanon',
+        'LS': 'Lesotho',
+        'LR': 'Liberia',
+        'LY': 'Libya',
+        'LI': 'Liechtenstein',
+        'LT': 'Lithuania',
+        'LU': 'Luxembourg',
+        'MO': 'Macao',
+        'MK': 'Macedonia, the Former Yugoslav Republic of',
+        'MG': 'Madagascar',
+        'MW': 'Malawi',
+        'MY': 'Malaysia',
+        'MV': 'Maldives',
+        'ML': 'Mali',
+        'MT': 'Malta',
+        'MH': 'Marshall Islands',
+        'MQ': 'Martinique',
+        'MR': 'Mauritania',
+        'MU': 'Mauritius',
+        'YT': 'Mayotte',
+        'MX': 'Mexico',
+        'FM': 'Micronesia, Federated States of',
+        'MD': 'Moldova, Republic of',
+        'MC': 'Monaco',
+        'MN': 'Mongolia',
+        'ME': 'Montenegro',
+        'MS': 'Montserrat',
+        'MA': 'Morocco',
+        'MZ': 'Mozambique',
+        'MM': 'Myanmar',
+        'NA': 'Namibia',
+        'NR': 'Nauru',
+        'NP': 'Nepal',
+        'NL': 'Netherlands',
+        'NC': 'New Caledonia',
+        'NZ': 'New Zealand',
+        'NI': 'Nicaragua',
+        'NE': 'Niger',
+        'NG': 'Nigeria',
+        'NU': 'Niue',
+        'NF': 'Norfolk Island',
+        'MP': 'Northern Mariana Islands',
+        'NO': 'Norway',
+        'OM': 'Oman',
+        'PK': 'Pakistan',
+        'PW': 'Palau',
+        'PS': 'Palestine, State of',
+        'PA': 'Panama',
+        'PG': 'Papua New Guinea',
+        'PY': 'Paraguay',
+        'PE': 'Peru',
+        'PH': 'Philippines',
+        'PN': 'Pitcairn',
+        'PL': 'Poland',
+        'PT': 'Portugal',
+        'PR': 'Puerto Rico',
+        'QA': 'Qatar',
+        'RE': 'Réunion',
+        'RO': 'Romania',
+        'RU': 'Russian Federation',
+        'RW': 'Rwanda',
+        'BL': 'Saint Barthélemy',
+        'SH': 'Saint Helena, Ascension and Tristan da Cunha',
+        'KN': 'Saint Kitts and Nevis',
+        'LC': 'Saint Lucia',
+        'MF': 'Saint Martin (French part)',
+        'PM': 'Saint Pierre and Miquelon',
+        'VC': 'Saint Vincent and the Grenadines',
+        'WS': 'Samoa',
+        'SM': 'San Marino',
+        'ST': 'Sao Tome and Principe',
+        'SA': 'Saudi Arabia',
+        'SN': 'Senegal',
+        'RS': 'Serbia',
+        'SC': 'Seychelles',
+        'SL': 'Sierra Leone',
+        'SG': 'Singapore',
+        'SX': 'Sint Maarten (Dutch part)',
+        'SK': 'Slovakia',
+        'SI': 'Slovenia',
+        'SB': 'Solomon Islands',
+        'SO': 'Somalia',
+        'ZA': 'South Africa',
+        'GS': 'South Georgia and the South Sandwich Islands',
+        'SS': 'South Sudan',
+        'ES': 'Spain',
+        'LK': 'Sri Lanka',
+        'SD': 'Sudan',
+        'SR': 'Suriname',
+        'SJ': 'Svalbard and Jan Mayen',
+        'SZ': 'Swaziland',
+        'SE': 'Sweden',
+        'CH': 'Switzerland',
+        'SY': 'Syrian Arab Republic',
+        'TW': 'Taiwan, Province of China',
+        'TJ': 'Tajikistan',
+        'TZ': 'Tanzania, United Republic of',
+        'TH': 'Thailand',
+        'TL': 'Timor-Leste',
+        'TG': 'Togo',
+        'TK': 'Tokelau',
+        'TO': 'Tonga',
+        'TT': 'Trinidad and Tobago',
+        'TN': 'Tunisia',
+        'TR': 'Turkey',
+        'TM': 'Turkmenistan',
+        'TC': 'Turks and Caicos Islands',
+        'TV': 'Tuvalu',
+        'UG': 'Uganda',
+        'UA': 'Ukraine',
+        'AE': 'United Arab Emirates',
+        'GB': 'United Kingdom',
+        'US': 'United States',
+        'UM': 'United States Minor Outlying Islands',
+        'UY': 'Uruguay',
+        'UZ': 'Uzbekistan',
+        'VU': 'Vanuatu',
+        'VE': 'Venezuela, Bolivarian Republic of',
+        'VN': 'Viet Nam',
+        'VG': 'Virgin Islands, British',
+        'VI': 'Virgin Islands, U.S.',
+        'WF': 'Wallis and Futuna',
+        'EH': 'Western Sahara',
+        'YE': 'Yemen',
+        'ZM': 'Zambia',
+        'ZW': 'Zimbabwe',
+    }
+
+    @classmethod
+    def short2full(cls, code):
+        """Convert an ISO 3166-2 country code to the corresponding full name"""
+        return cls._country_map.get(code.upper())
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From af214c3a79bb598d3a13559c5ebddcca4584669a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Jun 2015 13:15:57 +0800
Subject: [PATCH 0507/2145] [youtube] More useful messages for georestricted
 videos (#5716)

---
 youtube_dl/extractor/youtube.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..d9240ff02 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -32,6 +32,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     uppercase_escape,
+    ISO3166Utils,
 )
 
 
@@ -903,6 +904,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         break
         if 'token' not in video_info:
             if 'reason' in video_info:
+                if 'The uploader has not made this video available in your country.' in video_info['reason']:
+                    regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
+                    if regions_allowed is not None:
+                        raise ExtractorError('YouTube said: This video is available in %s only' % (
+                            ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
+                            expected=True)
                 raise ExtractorError(
                     'YouTube said: %s' % video_info['reason'][0],
                     expected=True, video_id=video_id)

From bc93bdb5bbf48b121977a5088ab9607f0fbeb83e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 13:19:46 +0600
Subject: [PATCH 0508/2145] [youtube] Fix reference before assignment for
 video_info

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 46841617a..45e5cb80e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -883,6 +883,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             add_dash_mpd(video_info)
         else:
             age_gate = False
+            video_info = None
             # Try looking directly into the video webpage
             mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
             if mobj:

From 226b886ca85b5843d2cb9638fb6d88223e9dd5b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:04:55 +0600
Subject: [PATCH 0509/2145] [vk] Fix authentication (Closes #6105)

---
 youtube_dl/extractor/vk.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 38ff3c1a9..f2ae109f9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -121,20 +121,27 @@ class VKIE(InfoExtractor):
         if username is None:
             return
 
-        login_form = {
-            'act': 'login',
-            'role': 'al_frame',
-            'expire': '1',
+        login_page = self._download_webpage(
+            'https://vk.com', None, 'Downloading login page')
+
+        login_form = dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            login_page))
+
+        login_form.update({
             'email': username.encode('cp1251'),
             'pass': password.encode('cp1251'),
-        }
+        })
 
-        request = compat_urllib_request.Request('https://login.vk.com/?act=login',
-                                                compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+        request = compat_urllib_request.Request(
+            'https://login.vk.com/?act=login',
+            compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        login_page = self._download_webpage(
+            request, None, note='Logging in as %s' % username)
 
         if re.search(r'onLoginFailed', login_page):
-            raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
+            raise ExtractorError(
+                'Unable to login, incorrect username and/or password', expected=True)
 
     def _real_initialize(self):
         self._login()

From 00334d0de01d46165880569159c534f9b7d927a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:26:51 +0600
Subject: [PATCH 0510/2145] [options] Add missing whitespace and split lines

---
 youtube_dl/options.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6aeca61ee..e7d067642 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -351,7 +351,8 @@ def parseOpts(overrideArguments=None):
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,
         help=(
-            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.'
+            'If a merge is required (e.g. bestvideo+bestaudio), '
+            'output to given container format. One of mkv, mp4, ogg, webm, flv. '
             'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')

From 0a3cf9ad3dc1a038018ff347473c0e2d8fc10a69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:31:18 +0600
Subject: [PATCH 0511/2145] [youtube] Skip get_video_info requests when
 --youtube-skip-dash-manifest is specified

---
 youtube_dl/extractor/youtube.py | 43 +++++++++++++++++----------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 45e5cb80e..2c9ad5e92 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -894,27 +894,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
-            # We also try looking in get_video_info since it may contain different dashmpd
-            # URL that points to a DASH manifest with possibly different itag set (some itags
-            # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
-            # manifest pointed by get_video_info's dashmpd).
-            # The general idea is to take a union of itags of both DASH manifests (for example
-            # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
-            self.report_video_info_webpage_download(video_id)
-            for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                video_info_url = (
-                    '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                    % (proto, video_id, el_type))
-                video_info_webpage = self._download_webpage(
-                    video_info_url,
-                    video_id, note=False,
-                    errnote='unable to download video info webpage')
-                get_video_info = compat_parse_qs(video_info_webpage)
-                add_dash_mpd(get_video_info)
-                if not video_info:
-                    video_info = get_video_info
-                if 'token' in get_video_info:
-                    break
+            if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
+                # We also try looking in get_video_info since it may contain different dashmpd
+                # URL that points to a DASH manifest with possibly different itag set (some itags
+                # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
+                # manifest pointed by get_video_info's dashmpd).
+                # The general idea is to take a union of itags of both DASH manifests (for example
+                # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+                self.report_video_info_webpage_download(video_id)
+                for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                    video_info_url = (
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (proto, video_id, el_type))
+                    video_info_webpage = self._download_webpage(
+                        video_info_url,
+                        video_id, note=False,
+                        errnote='unable to download video info webpage')
+                    get_video_info = compat_parse_qs(video_info_webpage)
+                    add_dash_mpd(get_video_info)
+                    if not video_info:
+                        video_info = get_video_info
+                    if 'token' in get_video_info:
+                        break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(

From b2575b38e76e14b7552a43cd707a03fe78748f40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:38:41 +0600
Subject: [PATCH 0512/2145] [options] Clarify --youtube-skip-dash-manifest

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6aeca61ee..e3dfb7af9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -346,7 +346,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--youtube-skip-dash-manifest',
         action='store_false', dest='youtube_include_dash_manifest',
-        help='Do not download the DASH manifest on YouTube videos')
+        help='Do not download the DASH manifests and related data on YouTube videos')
     video_format.add_option(
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,

From da77d856a1310b52975abbad82121a1b4c6597a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:55:46 +0600
Subject: [PATCH 0513/2145] [youtube] Add test for #6093

---
 youtube_dl/extractor/youtube.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2c9ad5e92..20e1781f8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -518,6 +518,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
+        {
+            'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
+            'info_dict': {
+                'id': 'FIl7x6_3R5Y',
+                'ext': 'mp4',
+                'title': 'md5:7b81415841e02ecd4313668cde88737a',
+                'description': 'md5:116377fd2963b81ec4ce64b542173306',
+                'upload_date': '20150625',
+                'uploader_id': 'dorappi2000',
+                'uploader': 'dorappi2000',
+                'formats': 'mincount:33',
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From f39eb98bab497d8e06b9f243a8240509326678f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 27 Jun 2015 10:55:25 +0100
Subject: [PATCH 0514/2145] download all pages before start extracting info

---
 youtube_dl/extractor/snagfilms.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cd345474e..74cd2698d 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -34,7 +34,6 @@ class SnagFilmsIE(InfoExtractor):
         display_id, video_id = match(self._VALID_URL,url).groups()
         if display_id is None:
             embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-
             display_id = self._html_search_regex(
                 r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
                 embed_webpage,
@@ -48,16 +47,16 @@ class SnagFilmsIE(InfoExtractor):
             'data'
         ), display_id)
 
+        if video_id is None:
+            video_id = json_data['id']
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+
         title = json_data['title']
         duration = int(json_data['duration'])
         description = json_data['synopsis']
         categories = [category['title'] for category in json_data['categories']]
         thumbnail = json_data['image']
 
-        if video_id is None:
-            video_id = json_data['id']
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-
         sources = self._parse_json(js_to_json(self._html_search_regex(
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,

From 73c471e9efd3f888af060b391eb750b079a5822c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Jun 2015 14:18:01 +0200
Subject: [PATCH 0515/2145] [newstube] Fix GUID extraction (fixes #6109)

---
 youtube_dl/extractor/newstube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index 85fcad06b..fe29cfc3f 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -31,7 +31,7 @@ class NewstubeIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         video_guid = self._html_search_regex(
-            r'<meta property="og:video" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+                r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             page, 'video GUID')
 
         player = self._download_xml(

From cee16e0fa3c2030aa6e58669ca2253457f1f3e64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Jun 2015 14:20:33 +0200
Subject: [PATCH 0516/2145] [newstube] style: fix alignment

---
 youtube_dl/extractor/newstube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index fe29cfc3f..5a9e73cd6 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -31,7 +31,7 @@ class NewstubeIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         video_guid = self._html_search_regex(
-                r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             page, 'video GUID')
 
         player = self._download_xml(

From 654fd03c73fa0e4407a71c07d821b45321c3cdb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:20:42 +0600
Subject: [PATCH 0517/2145] [snagfilms] Improve and simplify

---
 youtube_dl/extractor/__init__.py  |   5 +-
 youtube_dl/extractor/snagfilms.py | 171 +++++++++++++++++++-----------
 2 files changed, 114 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 59068a8b8..7e74a971d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -493,7 +493,10 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
-from .snagfilms import SnagFilmsIE
+from .snagfilms import (
+    SnagFilmsIE,
+    SnagFilmsEmbedIE,
+)
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soompi import (
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 74cd2698d..6e103bd49 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -1,84 +1,133 @@
-from re import match,DOTALL
+from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    parse_duration,
+)
+
+
+class SnagFilmsEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|embed)\.)?snagfilms\.com/embed/player\?.*\bfilmId=(?P<id>[\da-f-]{36})'
+    _TESTS = [{
+        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
+        'md5': '2924e9215c6eff7a55ed35b72276bd93',
+        'info_dict': {
+            'id': '74849a00-85a9-11e1-9660-123139220831',
+            'ext': 'mp4',
+            'title': '#whilewewatch',
+        }
+    }, {
+        'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for source in self._parse_json(js_to_json(self._search_regex(
+                r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):
+            file_ = source.get('file')
+            if not file_:
+                continue
+            type_ = source.get('type')
+            format_id = source.get('label')
+            ext = determine_ext(file_)
+            if any(_ == 'm3u8' for _ in (type_, ext)):
+                formats.extend(self._extract_m3u8_formats(
+                    file_, video_id, 'mp4', m3u8_id='hls'))
+            else:
+                bitrate = int_or_none(self._search_regex(
+                    r'(\d+)kbps', file_, 'bitrate', default=None))
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]$', format_id, 'height', default=None))
+                formats.append({
+                    'url': file_,
+                    'format_id': format_id,
+                    'tbr': bitrate,
+                    'height': height,
+                })
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
+            webpage, 'title')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }
+
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www.|embed.)?snagfilms\.com/(?:films/title/(?P<display_id>.+?)|embed/player\?.*filmId=(?P<id>.+?))(?:&|/|$)'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/films/title/(?P<id>[^/]+)'
+    _TEST = {
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
-        'info_dict':
-        {
+        'md5': '19844f897b35af219773fd63bdec2942',
+        'info_dict': {
             'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
             'display_id': 'lost_for_life',
             'ext': 'mp4',
             'title': 'Lost for Life',
+            'description': 'md5:fbdacc8bb6b455e464aaf98bc02e1c82',
+            'thumbnail': 're:^https?://.*\.jpg',
             'duration': 4489,
-            'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
-            'categories': ['Documentary','Crime','Award Winning','Festivals']
+            'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
         }
-    },{
-        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831',
-        'info_dict':
-        {
-            'id': '74849a00-85a9-11e1-9660-123139220831',
-            'display_id': 'while_we_watch',
-            'ext': 'mp4',
-            'title': '#whilewewatch',
-            'duration': 2311,
-            'description': 'A gripping portrait of the Occupy Wall Street media revolution,&nbsp;#WHILEWEWATCH is the first definitive film to emerge from Zuccotti Park—with full access and cooperation from masterminds who made #OccupyWallStreet a reality.&nbsp;The #OccupyWallStreet media team had no fear of a critical city government, big corporations, hostile police or a lagging mainstream media to tell their story. Through rain, snow, grueling days and sleeping on concrete, they pump out exhilarating ideas to the world. With little money, they rely on Twitter, texting, Wi-Fi, posters, Tumblr, live streams, YouTube, Facebook, dramatic marches, drumbeats and chants. As the film unfolds, we witness the burgeoning power of social media.<br />',
-            'categories': ['Documentary','Politics']
-        }
-    }]
+    }
 
     def _real_extract(self, url):
-        display_id, video_id = match(self._VALID_URL,url).groups()
-        if display_id is None:
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-            display_id = self._html_search_regex(
-                r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
-                embed_webpage,
-                'display_id'
-            )
-        webpage = self._download_webpage('http://www.snagfilms.com/films/title/' + display_id, display_id)
+        display_id = self._match_id(url)
 
-        json_data = self._parse_json(self._html_search_regex(
-            r'"data":{"film":(?P<data>{.*?}})}',
-            webpage,
-            'data'
-        ), display_id)
+        webpage = self._download_webpage(url, display_id)
 
-        if video_id is None:
-            video_id = json_data['id']
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+        film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 
-        title = json_data['title']
-        duration = int(json_data['duration'])
-        description = json_data['synopsis']
-        categories = [category['title'] for category in json_data['categories']]
-        thumbnail = json_data['image']
+        snag = self._parse_json(
+            self._search_regex(
+                'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag'),
+            display_id)
 
-        sources = self._parse_json(js_to_json(self._html_search_regex(
-            r'sources: (?P<sources>\[.*?\])',
-            embed_webpage,
-            'sources',
-            flags=DOTALL
-        )), video_id)
-
-        formats = []
-        for source in sources:
-            if source['type'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
-            else:
-                formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
-        self._sort_formats(formats)
+        for item in snag:
+            if item.get('data', {}).get('film', {}).get('id') == film_id:
+                data = item['data']['film']
+                title = data['title']
+                description = clean_html(data.get('synopsis'))
+                thumbnail = data.get('image')
+                duration = int_or_none(data.get('duration') or data.get('runtime'))
+                categories = [
+                    category['title'] for category in data.get('categories', [])
+                    if category.get('title')]
+                break
+        else:
+            title = self._search_regex(
+                r'itemprop="title">([^<]+)<', webpage, 'title')
+            description = self._html_search_regex(
+                r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
+                webpage, 'description', default=None) or self._og_search_description(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
+            duration = parse_duration(self._search_regex(
+                r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
+                webpage, 'duration', fatal=False))
+            categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'url': 'http://embed.snagfilms.com/embed/player?filmId=%s' % film_id,
+            'id': film_id,
             'display_id': display_id,
             'title': title,
-            'duration': duration,
             'description': description,
-            'categories': categories,
             'thumbnail': thumbnail,
-            'formats': formats,
+            'duration': duration,
+            'categories': categories,
         }

From 7c197ad96dd0f36177eda66777c93502228fc36b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:25:50 +0600
Subject: [PATCH 0518/2145] [snagfilms] Add routine for generic embeds
 extractions

---
 youtube_dl/extractor/snagfilms.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 6e103bd49..cb52eb72b 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -27,6 +27,13 @@ class SnagFilmsEmbedIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From eedd20ef9637660d5585b1b7d221d7f2d31630ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:26:14 +0600
Subject: [PATCH 0519/2145] [extractor/generic] Add support for snagfilms
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 42e4e7035..5b1da47e3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -47,6 +47,7 @@ from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
+from .snagfilms import SnagFilmsEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1550,6 +1551,11 @@ class GenericIE(InfoExtractor):
         if onionstudios_url:
             return self.url_result(onionstudios_url)
 
+        # Look for SnagFilms embeds
+        snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
+        if snagfilms_url:
+            return self.url_result(snagfilms_url)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From b8c1cc1a51c16682725cf382f0e498a390c62a2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:28:10 +0600
Subject: [PATCH 0520/2145] [extractor/generic] Add test for snagfilms embeds

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5b1da47e3..32e41d13e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -850,6 +850,15 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'clickhole',
             }
         },
+        # SnagFilms embed
+        {
+            'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
+            'info_dict': {
+                'id': '74849a00-85a9-11e1-9660-123139220831',
+                'ext': 'mp4',
+                'title': '#whilewewatch',
+            }
+        },
         # AdobeTVVideo embed
         {
             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',

From 242a998bdc5e808411870d7b13395b7739501f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:40:01 +0600
Subject: [PATCH 0521/2145] [snagfilms] Add support for shows

---
 youtube_dl/extractor/snagfilms.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cb52eb72b..f04ca6036 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -76,8 +76,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/films/title/(?P<id>[^/]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/(?:films/title|show)/(?P<id>[^?#]+)'
+    _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'md5': '19844f897b35af219773fd63bdec2942',
         'info_dict': {
@@ -90,7 +90,20 @@ class SnagFilmsIE(InfoExtractor):
             'duration': 4489,
             'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
         }
-    }
+    }, {
+        'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
+        'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
+        'info_dict': {
+            'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
+            'display_id': 'the_world_cut_project/india',
+            'ext': 'mp4',
+            'title': 'India',
+            'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 979,
+            'categories': ['Documentary', 'Sports', 'Politics']
+        }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From 9fbfc9bd4dd076e2b00f68dc43ce4fa0ed72c949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:50:26 +0600
Subject: [PATCH 0522/2145] [snagfilms:embed] Capture geolocation restriction
 error

---
 youtube_dl/extractor/snagfilms.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index f04ca6036..2b1838bf9 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     determine_ext,
     int_or_none,
@@ -30,7 +31,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1',
+            webpage)
         if mobj:
             return mobj.group('url')
 
@@ -39,6 +41,10 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        if '>This film is not playable in your area.<' in webpage:
+            raise ExtractorError(
+                'This film is not playable in your area.', expected=True)
+
         formats = []
         for source in self._parse_json(js_to_json(self._search_regex(
                 r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):

From a42a1bb09dd1c86b4d35eedf1104d8e51b6c407b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:54:08 +0600
Subject: [PATCH 0523/2145] [snagfilms] Capture not available error

---
 youtube_dl/extractor/snagfilms.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 2b1838bf9..1be7dd13b 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -43,7 +43,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
         if '>This film is not playable in your area.<' in webpage:
             raise ExtractorError(
-                'This film is not playable in your area.', expected=True)
+                'Film %s is not playable in your area.' % video_id, expected=True)
 
         formats = []
         for source in self._parse_json(js_to_json(self._search_regex(
@@ -116,6 +116,10 @@ class SnagFilmsIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        if ">Sorry, the Film you're looking for is not available.<" in webpage:
+            raise ExtractorError(
+                'Film %s is not available.' % display_id, expected=True)
+
         film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 
         snag = self._parse_json(

From a9de9517445a4971cb89c73d28a3c7b9ecf51b11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:57:01 +0600
Subject: [PATCH 0524/2145] [snagfilms] More tests

---
 youtube_dl/extractor/snagfilms.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 1be7dd13b..cf495f310 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -109,6 +109,14 @@ class SnagFilmsIE(InfoExtractor):
             'duration': 979,
             'categories': ['Documentary', 'Sports', 'Politics']
         }
+    }, {
+        # Film is not playable in your area.
+        'url': 'http://www.snagfilms.com/films/title/inside_mecca',
+        'only_matching': True,
+    }, {
+        # Film is not available.
+        'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 81103ef35d5148f52229f35e91b99ff8d9218b10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:00:27 +0600
Subject: [PATCH 0525/2145] [smotri] Fix password protected video extraction

---
 youtube_dl/extractor/smotri.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 24746a09a..b8fa0457d 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -152,6 +152,10 @@ class SmotriIE(InfoExtractor):
             'getvideoinfo': '1',
         }
 
+        video_password = self._downloader.params.get('videopassword', None)
+        if video_password:
+            video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
+
         request = compat_urllib_request.Request(
             'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@@ -161,13 +165,18 @@ class SmotriIE(InfoExtractor):
         video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
 
         if not video_url:
-            if video.get('_moderate_no') or not video.get('moderated'):
+            if video.get('_moderate_no'):
                 raise ExtractorError(
                     'Video %s has not been approved by moderator' % video_id, expected=True)
 
             if video.get('error'):
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
+            if video.get('_pass_protected') == 1:
+                msg = ('Invalid video password' if video_password
+                       else 'This video is protected by a password, use the --video-password option')
+                raise ExtractorError(msg, expected=True)
+
         title = video['title']
         thumbnail = video['_imgURL']
         upload_date = unified_strdate(video['added'])

From c3b7202f4fdb8a78a90a96f3cf6213f6ac215b3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:03:26 +0600
Subject: [PATCH 0526/2145] [smotri] Remove non relevant test

---
 youtube_dl/extractor/smotri.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index b8fa0457d..cc8dce3db 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -90,21 +90,6 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
-        # not approved by moderator, but available
-        {
-            'url': 'http://smotri.com/video/view/?id=v28888533b73',
-            'md5': 'f44bc7adac90af518ef1ecf04893bb34',
-            'info_dict': {
-                'id': 'v28888533b73',
-                'ext': 'mp4',
-                'title': 'Russian Spies Killed By ISIL Child Soldier',
-                'uploader': 'Mopeder',
-                'uploader_id': 'mopeder',
-                'duration': 71,
-                'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
-                'upload_date': '20150114',
-            },
-        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',

From 04c27802c08268c5c9e42ca226d2a848a657b82c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:31:27 +0600
Subject: [PATCH 0527/2145] [smotri] Add tests for password protected videos

---
 youtube_dl/extractor/smotri.py | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index cc8dce3db..93a7cfe15 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -53,7 +53,7 @@ class SmotriIE(InfoExtractor):
                 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
             },
         },
-        # video-password
+        # video-password, not approved by moderator
         {
             'url': 'http://smotri.com/video/view/?id=v1390466a13c',
             'md5': 'f6331cef33cad65a0815ee482a54440b',
@@ -71,7 +71,24 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
-        # age limit + video-password
+        # video-password
+        {
+            'url': 'http://smotri.com/video/view/?id=v6984858774#',
+            'md5': 'f11e01d13ac676370fc3b95b9bda11b0',
+            'info_dict': {
+                'id': 'v6984858774',
+                'ext': 'mp4',
+                'title': 'Дача Солженицина ПАРОЛЬ 223322',
+                'uploader': 'psavari1',
+                'uploader_id': 'psavari1',
+                'upload_date': '20081103',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'videopassword': '223322',
+            },
+        },
+        # age limit + video-password, not approved by moderator
         {
             'url': 'http://smotri.com/video/view/?id=v15408898bcf',
             'md5': '91e909c9f0521adf5ee86fbe073aad70',
@@ -90,6 +107,24 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
+        # age limit + video-password
+        {
+            'url': 'http://smotri.com/video/view/?id=v7780025814',
+            'md5': 'b4599b068422559374a59300c5337d72',
+            'info_dict': {
+                'id': 'v7780025814',
+                'ext': 'mp4',
+                'title': 'Sexy Beach (пароль 123)',
+                'uploader': 'вАся',
+                'uploader_id': 'asya_prosto',
+                'upload_date': '20081218',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'age_limit': 18,
+            },
+            'params': {
+                'videopassword': '123'
+            },
+        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',

From 6c4d20cd6fae8e47c1f0c6771e2e91660bf10eb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 00:08:52 +0600
Subject: [PATCH 0528/2145] [downloader/external] Fix externals downloaders
 specified with extension on Windows

---
 youtube_dl/downloader/external.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index a57c15856..1d5cc9904 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -131,5 +131,6 @@ def list_external_downloaders():
 def get_external_downloader(external_downloader):
     """ Given the name of the executable, see whether we support the given
         downloader . """
-    bn = os.path.basename(external_downloader)
+    # Drop .exe extension on Windows
+    bn = os.path.splitext(os.path.basename(external_downloader))[0]
     return _BY_NAME[bn]

From 82ea1051b54134150f290d3d9f259a4baf54faf4 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 05:11:23 +0100
Subject: [PATCH 0529/2145] [moviefap] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/moviefap.py | 117 +++++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 youtube_dl/extractor/moviefap.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7e74a971d..b70c1b57e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -311,6 +311,7 @@ from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
+from .moviefap import MovieFapIE
 from .moviezine import MoviezineIE
 from .movshare import MovShareIE
 from .mtv import (
diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
new file mode 100644
index 000000000..49b8ab7d9
--- /dev/null
+++ b/youtube_dl/extractor/moviefap.py
@@ -0,0 +1,117 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import str_to_int
+
+
+class MovieFapIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
+    _TESTS = [{
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
+            'uploader_id': 'whiskeyjar',
+            'display_id': 'jeune-couple-russe'
+        }
+    }, {
+        'url': 'http://www.moviefap.com/videos/3080837f6712355015c2/busty-british-blonde-takes-backdoor-in-fake-taxi.html',
+        'md5': 'bedef72cb23d27a20755fc430a6d7a0e',
+        'info_dict': {
+            'id': '3080837f6712355015c2',
+            'ext': 'mp4',
+            'title': 'Busty British blonde takes backdoor in fake taxi',
+            'description': 'Big boobs British blonde flashing in fake taxi then giving titsjob and rimjob in the back seat before getting big cock up her tight ass',
+            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/30/322021-18l.jpg',
+            'uploader_id': 'momcikoper',
+            'display_id': 'busty-british-blonde-takes-backdoor-in-fake-taxi'
+        }
+    }]
+
+    @staticmethod
+    def __get_thumbnail_data(xml):
+
+        """
+        Constructs a list of video thumbnails from timeline preview images.
+        :param xml: the information XML document to parse
+        """
+
+        timeline = xml.find('timeline')
+        if timeline is None:
+            # not all videos have the data - ah well
+            return []
+
+        # get the required information from the XML
+        attrs = {attr: str_to_int(timeline.find(attr).text)
+                 for attr in ['imageWidth', 'imageHeight', 'imageFirst', 'imageLast']}
+        pattern = timeline.find('imagePattern').text
+
+        # generate the list of thumbnail information dicts
+        thumbnails = []
+        for i in range(attrs['imageFirst'], attrs['imageLast'] + 1):
+            thumbnails.append({
+                'url': pattern.replace('#', str(i)),
+                'width': attrs['imageWidth'],
+                'height': attrs['imageHeight']
+            })
+        return thumbnails
+
+    def _real_extract(self, url):
+
+        # find the video ID
+        video_id = self._match_id(url)
+
+        # retrieve the page HTML
+        webpage = self._download_webpage(url, video_id)
+
+        # find the URL of the XML document detailing video download URLs
+        info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+
+        # download that XML
+        xml = self._download_xml(info_url, video_id)
+
+        # create dictionary of properties we know so far, or can find easily
+        info = {
+            'id': video_id,
+            'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'display_id': re.compile(self._VALID_URL).match(url).group('name'),
+            'thumbnails': self.__get_thumbnail_data(xml),
+            'thumbnail': xml.find('startThumb').text,
+            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description'),
+            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id'),
+            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count')),
+            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating')),
+            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count')),
+            'age_limit': 18,
+            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url'),
+            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories').split(', ')
+        }
+
+        # find and add the format
+        if xml.find('videoConfig') is not None:
+            info['ext'] = xml.find('videoConfig').find('type').text
+        else:
+            info['ext'] = 'flv'  # guess...
+
+        # work out the video URL(s)
+        if xml.find('videoLink') is not None:
+            # single format available
+            info['url'] = xml.find('videoLink').text
+        else:
+            # multiple formats available
+            info['formats'] = []
+
+            # N.B. formats are already in ascending order of quality
+            for item in xml.find('quality').findall('item'):
+                info['formats'].append({
+                    'url': item.find('videoLink').text,
+                    'resolution': item.find('res').text  # 480p etc.
+                })
+
+        return info

From 71f9e49e67bfe62c9f1a6d8f74b7d81ab820ba84 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 05:22:35 +0100
Subject: [PATCH 0530/2145] [moviefap] Fix dictionary comprehension syntax
 incompatible with Python 2.6

---
 youtube_dl/extractor/moviefap.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 49b8ab7d9..88f9dab6f 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -48,17 +48,19 @@ class MovieFapIE(InfoExtractor):
             return []
 
         # get the required information from the XML
-        attrs = {attr: str_to_int(timeline.find(attr).text)
-                 for attr in ['imageWidth', 'imageHeight', 'imageFirst', 'imageLast']}
+        width = str_to_int(timeline.find('imageWidth').text)
+        height = str_to_int(timeline.find('imageHeight').text)
+        first = str_to_int(timeline.find('imageFirst').text)
+        last = str_to_int(timeline.find('imageLast').text)
         pattern = timeline.find('imagePattern').text
 
         # generate the list of thumbnail information dicts
         thumbnails = []
-        for i in range(attrs['imageFirst'], attrs['imageLast'] + 1):
+        for i in range(first, last + 1):
             thumbnails.append({
                 'url': pattern.replace('#', str(i)),
-                'width': attrs['imageWidth'],
-                'height': attrs['imageHeight']
+                'width': width,
+                'height': height
             })
         return thumbnails
 

From 802d74aa6ba2613f95d19c3be6c2480b9a2ebe5b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:01:27 +0100
Subject: [PATCH 0531/2145] [moviefap] Swap test for an alternative
 non-copyrighted video

---
 youtube_dl/extractor/moviefap.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 88f9dab6f..880ea2247 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -21,16 +21,16 @@ class MovieFapIE(InfoExtractor):
             'display_id': 'jeune-couple-russe'
         }
     }, {
-        'url': 'http://www.moviefap.com/videos/3080837f6712355015c2/busty-british-blonde-takes-backdoor-in-fake-taxi.html',
-        'md5': 'bedef72cb23d27a20755fc430a6d7a0e',
+        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
+        'md5': '26624b4e2523051b550067d547615906',
         'info_dict': {
-            'id': '3080837f6712355015c2',
+            'id': 'be9867c9416c19f54a4a',
             'ext': 'mp4',
-            'title': 'Busty British blonde takes backdoor in fake taxi',
-            'description': 'Big boobs British blonde flashing in fake taxi then giving titsjob and rimjob in the back seat before getting big cock up her tight ass',
-            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/30/322021-18l.jpg',
-            'uploader_id': 'momcikoper',
-            'display_id': 'busty-british-blonde-takes-backdoor-in-fake-taxi'
+            'title': 'Experienced MILF Amazing Handjob',
+            'description': 'Experienced MILF giving an Amazing Handjob',
+            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
+            'uploader_id': 'darvinfred06',
+            'display_id': 'experienced-milf-amazing-handjob'
         }
     }]
 

From 9c494108983c2a951780217014c26d8f3b081682 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:06:44 +0100
Subject: [PATCH 0532/2145] [moviefap] Add categories to tests

---
 youtube_dl/extractor/moviefap.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 880ea2247..6b79183c7 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -18,7 +18,8 @@ class MovieFapIE(InfoExtractor):
             'description': 'Amateur',
             'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
             'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe'
+            'display_id': 'jeune-couple-russe',
+            'categories': ['Amateur', 'Teen']
         }
     }, {
         'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
@@ -30,7 +31,8 @@ class MovieFapIE(InfoExtractor):
             'description': 'Experienced MILF giving an Amazing Handjob',
             'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
             'uploader_id': 'darvinfred06',
-            'display_id': 'experienced-milf-amazing-handjob'
+            'display_id': 'experienced-milf-amazing-handjob',
+            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
         }
     }]
 

From a8e6f30d8ed13209f2e9815ae97b3edccd38561b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:14:15 +0100
Subject: [PATCH 0533/2145] [moviefap] Swap and justify tests

---
 youtube_dl/extractor/moviefap.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 6b79183c7..1985c53c0 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -9,19 +9,7 @@ from ..utils import str_to_int
 class MovieFapIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
     _TESTS = [{
-        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
-        'md5': 'fa56683e291fc80635907168a743c9ad',
-        'info_dict': {
-            'id': 'e5da0d3edce5404418f5',
-            'ext': 'flv',
-            'title': 'Jeune Couple Russe',
-            'description': 'Amateur',
-            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
-            'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe',
-            'categories': ['Amateur', 'Teen']
-        }
-    }, {
+        # normal, multi-format video
         'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
         'md5': '26624b4e2523051b550067d547615906',
         'info_dict': {
@@ -34,6 +22,20 @@ class MovieFapIE(InfoExtractor):
             'display_id': 'experienced-milf-amazing-handjob',
             'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
         }
+    }, {
+        # quirky single-format case where the extension is given as fid, but the video is really an flv
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
+            'uploader_id': 'whiskeyjar',
+            'display_id': 'jeune-couple-russe',
+            'categories': ['Amateur', 'Teen']
+        }
     }]
 
     @staticmethod

From 23e7cba87fbcec9aa2b1fbccf60c2d560df3d7ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 01:22:25 +0600
Subject: [PATCH 0534/2145] [twitter:card] Add extractor (#5239)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitter.py  | 72 ++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/twitter.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7e74a971d..c0a825e8a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -621,6 +621,7 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
+from .twitter import TwitterCardIE
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
new file mode 100644
index 000000000..1aaa06305
--- /dev/null
+++ b/youtube_dl/extractor/twitter.py
@@ -0,0 +1,72 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    float_or_none,
+    unescapeHTML,
+)
+
+
+class TwitterCardIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
+        'md5': 'a74f50b310c83170319ba16de6955192',
+        'info_dict': {
+            'id': '560070183650213889',
+            'ext': 'mp4',
+            'title': 'TwitterCard',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 30.033,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Different formats served for different User-Agents
+        USER_AGENTS = [
+            'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',  # mp4
+            'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',  # webm
+        ]
+
+        config = None
+        formats = []
+        for user_agent in USER_AGENTS:
+            request = compat_urllib_request.Request(url)
+            request.add_header('User-Agent', user_agent)
+            webpage = self._download_webpage(request, video_id)
+
+            config = self._parse_json(
+                unescapeHTML(self._search_regex(
+                    r'data-player-config="([^"]+)"', webpage, 'data player config')),
+                video_id)
+
+            video_url = config['playlist'][0]['source']
+
+            f = {
+                'url': video_url,
+            }
+
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                f.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+            formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnail = config.get('posterImageUrl')
+        duration = float_or_none(config.get('duration'))
+
+        return {
+            'id': video_id,
+            'title': 'TwitterCard',
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From d16ef949ca94ffb998c4db5a8367f0b367d659e3 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:36:46 +0100
Subject: [PATCH 0535/2145] [moviefap] Allow non-critical fields to change
 without breaking extraction

---
 youtube_dl/extractor/moviefap.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 1985c53c0..23575d30a 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -89,14 +89,14 @@ class MovieFapIE(InfoExtractor):
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xml.find('startThumb').text,
-            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description'),
-            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id'),
-            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count')),
-            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating')),
-            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count')),
+            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url'),
-            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories').split(', ')
+            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
 
         # find and add the format

From 62b742ece3ec6c7d7fd24898b5413b6b98a4ae8f Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:51:11 +0100
Subject: [PATCH 0536/2145] [moviefap] Remove redundant comments

---
 youtube_dl/extractor/moviefap.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 23575d30a..b38a8e71f 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -70,19 +70,13 @@ class MovieFapIE(InfoExtractor):
 
     def _real_extract(self, url):
 
-        # find the video ID
         video_id = self._match_id(url)
-
-        # retrieve the page HTML
         webpage = self._download_webpage(url, video_id)
 
-        # find the URL of the XML document detailing video download URLs
+        # find and retrieve the XML document detailing video download URLs
         info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
-
-        # download that XML
         xml = self._download_xml(info_url, video_id)
 
-        # create dictionary of properties we know so far, or can find easily
         info = {
             'id': video_id,
             'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),

From 43b925ce74efd0a011f7880dcdcc90f4cf3b8f4b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:52:12 +0100
Subject: [PATCH 0537/2145] [moviefap] Replace calls to `find()` with
 `util.xpath_text()`.

---
 youtube_dl/extractor/moviefap.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index b38a8e71f..6da93dbc9 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import str_to_int
+from ..utils import (
+    xpath_text,
+    str_to_int
+)
 
 
 class MovieFapIE(InfoExtractor):
@@ -82,7 +85,7 @@ class MovieFapIE(InfoExtractor):
             'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
-            'thumbnail': xml.find('startThumb').text,
+            'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
             'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
             'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
             'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
@@ -102,7 +105,7 @@ class MovieFapIE(InfoExtractor):
         # work out the video URL(s)
         if xml.find('videoLink') is not None:
             # single format available
-            info['url'] = xml.find('videoLink').text
+            info['url'] = xpath_text(xml, 'videoLink', 'url', True)
         else:
             # multiple formats available
             info['formats'] = []
@@ -110,8 +113,8 @@ class MovieFapIE(InfoExtractor):
             # N.B. formats are already in ascending order of quality
             for item in xml.find('quality').findall('item'):
                 info['formats'].append({
-                    'url': item.find('videoLink').text,
-                    'resolution': item.find('res').text  # 480p etc.
+                    'url': xpath_text(item, 'videoLink', 'url', True),
+                    'resolution': xpath_text(item, 'res', 'resolution', True)  # 480p etc.
                 })
 
         return info

From b971abe897ee17fed7e36868fdc8880f6b145d7b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 21:04:53 +0100
Subject: [PATCH 0538/2145] [moviefap] Replace call to `str()` with
 `compat.compat_str()`

---
 youtube_dl/extractor/moviefap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 6da93dbc9..20a78f3b2 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -7,6 +7,7 @@ from ..utils import (
     xpath_text,
     str_to_int
 )
+from ..compat import compat_str
 
 
 class MovieFapIE(InfoExtractor):
@@ -65,7 +66,7 @@ class MovieFapIE(InfoExtractor):
         thumbnails = []
         for i in range(first, last + 1):
             thumbnails.append({
-                'url': pattern.replace('#', str(i)),
+                'url': pattern.replace('#', compat_str(i)),
                 'width': width,
                 'height': height
             })

From 8a1b49ff19a8a1fdc2c30cf10cc0598ac9bc8819 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 22:27:06 +0100
Subject: [PATCH 0539/2145] [moviefap] Explicitly sort formats to handle
 possible site changes

---
 youtube_dl/extractor/moviefap.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 20a78f3b2..295bfe3f0 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -111,11 +111,14 @@ class MovieFapIE(InfoExtractor):
             # multiple formats available
             info['formats'] = []
 
-            # N.B. formats are already in ascending order of quality
             for item in xml.find('quality').findall('item'):
+                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
                 info['formats'].append({
                     'url': xpath_text(item, 'videoLink', 'url', True),
-                    'resolution': xpath_text(item, 'res', 'resolution', True)  # 480p etc.
+                    'resolution': resolution,
+                    'height': int(re.findall(r'\d+', resolution)[0])
                 })
 
+            self._sort_formats(info['formats'])
+
         return info

From 1a5fd4eebc2717b5173df50d65007f90cb05ee30 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 22:32:56 +0100
Subject: [PATCH 0540/2145] [moviefap] Wrap long lines

---
 youtube_dl/extractor/moviefap.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 295bfe3f0..9de052a99 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -78,23 +78,32 @@ class MovieFapIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+        info_url = self._html_search_regex( \
+                r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
         info = {
             'id': video_id,
-            'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'title': self._html_search_regex( \
+                    r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
+            'description': self._html_search_regex( \
+                    r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex( \
+                    r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex( \
+                    r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex( \
+                    r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex( \
+                    r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
+            'webpage_url': self._html_search_regex( \
+                    r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex( \
+                    r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
 
         # find and add the format

From 5a9cc19972fb3aae7a67470f65ec5cd30918f4e1 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 23:03:06 +0100
Subject: [PATCH 0541/2145] [moviefap] Move flv videos to formats in the
 metadata

---
 youtube_dl/extractor/moviefap.py | 56 +++++++++++++++++---------------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 9de052a99..5e0c701d4 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -82,8 +82,36 @@ class MovieFapIE(InfoExtractor):
                 r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
-        info = {
+        # find the video container
+        if xml.find('videoConfig') is not None:
+            ext = xml.find('videoConfig').find('type').text
+        else:
+            ext = 'flv'  # guess...
+
+        # work out the video URL(s)
+        formats = []
+        if xml.find('videoLink') is not None:
+            # single format available
+            formats.append({
+                'url': xpath_text(xml, 'videoLink', 'url', True),
+                'ext': ext
+            })
+        else:
+            # multiple formats available
+            for item in xml.find('quality').findall('item'):
+                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
+                formats.append({
+                    'url': xpath_text(item, 'videoLink', 'url', True),
+                    'ext': ext,
+                    'resolution': resolution,
+                    'height': int(re.findall(r'\d+', resolution)[0])
+                })
+
+            self._sort_formats(formats)
+
+        return {
             'id': video_id,
+            'formats': formats,
             'title': self._html_search_regex( \
                     r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
@@ -105,29 +133,3 @@ class MovieFapIE(InfoExtractor):
             'categories': self._html_search_regex( \
                     r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
-
-        # find and add the format
-        if xml.find('videoConfig') is not None:
-            info['ext'] = xml.find('videoConfig').find('type').text
-        else:
-            info['ext'] = 'flv'  # guess...
-
-        # work out the video URL(s)
-        if xml.find('videoLink') is not None:
-            # single format available
-            info['url'] = xpath_text(xml, 'videoLink', 'url', True)
-        else:
-            # multiple formats available
-            info['formats'] = []
-
-            for item in xml.find('quality').findall('item'):
-                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
-                info['formats'].append({
-                    'url': xpath_text(item, 'videoLink', 'url', True),
-                    'resolution': resolution,
-                    'height': int(re.findall(r'\d+', resolution)[0])
-                })
-
-            self._sort_formats(info['formats'])
-
-        return info

From db652ea186586e3eda5006ee096161b1a867c0d0 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 23:04:55 +0100
Subject: [PATCH 0542/2145] [moviefap] Fix `flake8` warnings introduced in
 1a5fd4e

---
 youtube_dl/extractor/moviefap.py | 36 ++++++++++++++++----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 5e0c701d4..82b863539 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -78,8 +78,8 @@ class MovieFapIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex( \
-                r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+        info_url = self._html_search_regex(
+            r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
         # find the video container
@@ -112,24 +112,24 @@ class MovieFapIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': self._html_search_regex( \
-                    r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'title': self._html_search_regex(
+                r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex( \
-                    r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex( \
-                    r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex( \
-                    r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex( \
-                    r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex( \
-                    r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
+            'description': self._html_search_regex(
+                r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex(
+                r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex(
+                r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex(
+                r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex(
+                r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex( \
-                    r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex( \
-                    r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
+            'webpage_url': self._html_search_regex(
+                r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex(
+                r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }

From bb512e57dc138b261cf9c71a833b0df5d5ba849f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Jun 2015 13:25:59 +0800
Subject: [PATCH 0543/2145] [twitch:vod] Fix 'Source' format in m3u8 (closes
 #6115)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 94bd6345d..3e798e62d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -215,7 +215,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         formats = self._extract_m3u8_formats(
-            '%s/vod/%s?nauth=%s&nauthsig=%s'
+            '%s/vod/%s?nauth=%s&nauthsig=%s&allow_source=true'
             % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
         self._prefer_source(formats)

From ac0474f89d3e6f8c8c1fb3223a16a18a2fd02bcb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Jun 2015 13:31:37 +0800
Subject: [PATCH 0544/2145] [twitch:vod] Update _TEST

The original test case is gone
---
 youtube_dl/extractor/twitch.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 3e798e62d..b56ee2959 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -189,17 +189,17 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_SHORTCUT = 'v'
 
     _TEST = {
-        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+        'url': 'http://www.twitch.tv/riotgames/v/6528877',
         'info_dict': {
-            'id': 'v3622000',
+            'id': 'v6528877',
             'ext': 'mp4',
-            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'title': 'LCK Summer Split - Week 6 Day 1',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 6951,
-            'timestamp': 1419028564,
-            'upload_date': '20141219',
-            'uploader': 'KSPTV',
-            'uploader_id': 'ksptv',
+            'duration': 17208,
+            'timestamp': 1435131709,
+            'upload_date': '20150624',
+            'uploader': 'Riot Games',
+            'uploader_id': 'riotgames',
             'view_count': int,
         },
         'params': {

From 9603e8a7d998615d3da1af47461ec9c353ec4e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:55:28 +0600
Subject: [PATCH 0545/2145] [YoutubeDL] Handle None width and height similarly
 to formats

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ef0f71bad..411de9ac9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1008,7 +1008,7 @@ class YoutubeDL(object):
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
             for i, t in enumerate(thumbnails):
-                if 'width' in t and 'height' in t:
+                if t.get('width') and t.get('height'):
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
                 if t.get('id') is None:
                     t['id'] = '%d' % i

From bf42a9906d9a066d32f1cc50e1b033e6676744ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:56:07 +0600
Subject: [PATCH 0546/2145] [utils] Add default value for xpath_text

---
 youtube_dl/utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 96490f112..942f76d24 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -62,6 +62,8 @@ std_headers = {
 }
 
 
+NO_DEFAULT = object()
+
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
@@ -171,13 +173,15 @@ def xpath_with_ns(path, ns_map):
     return '/'.join(replaced)
 
 
-def xpath_text(node, xpath, name=None, fatal=False):
+def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     if sys.version_info < (2, 7):  # Crazy 2.6
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
     if n is None or n.text is None:
-        if fatal:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
             name = xpath if name is None else name
             raise ExtractorError('Could not find XML element %s' % name)
         else:

From c342041fba9283ba5f05f48427aabf79adcf8647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:56:45 +0600
Subject: [PATCH 0547/2145] [extractor/common] Use NO_DEFAULT from utils

---
 youtube_dl/extractor/common.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 49e4dc710..7fa46d295 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -22,6 +22,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
+    NO_DEFAULT,
     age_restricted,
     bug_reports_message,
     clean_html,
@@ -33,7 +34,7 @@ from ..utils import (
     sanitize_filename,
     unescapeHTML,
 )
-_NO_DEFAULT = object()
+
 
 
 class InfoExtractor(object):
@@ -523,7 +524,7 @@ class InfoExtractor(object):
             video_info['description'] = playlist_description
         return video_info
 
-    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
+    def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Perform a regex search on the given string, using a single or a list of
         patterns returning the first matching group.
@@ -549,7 +550,7 @@ class InfoExtractor(object):
                 return next(g for g in mobj.groups() if g is not None)
             else:
                 return mobj.group(group)
-        elif default is not _NO_DEFAULT:
+        elif default is not NO_DEFAULT:
             return default
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
@@ -557,7 +558,7 @@ class InfoExtractor(object):
             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
-    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
+    def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Like _search_regex, but strips HTML tags and unescapes entities.
         """

From d16154d16327907279eff48a4018c495726d401a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 23:05:09 +0600
Subject: [PATCH 0548/2145] [tnaflix] Generalize tnaflix extractors

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/empflix.py  |  31 ----
 youtube_dl/extractor/moviefap.py | 135 ---------------
 youtube_dl/extractor/tnaflix.py  | 279 +++++++++++++++++++++++++------
 4 files changed, 234 insertions(+), 219 deletions(-)
 delete mode 100644 youtube_dl/extractor/empflix.py
 delete mode 100644 youtube_dl/extractor/moviefap.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d41d277c9..d44339200 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -144,7 +144,6 @@ from .ellentv import (
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE
-from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
@@ -311,7 +310,6 @@ from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
-from .moviefap import MovieFapIE
 from .moviezine import MoviezineIE
 from .movshare import MovShareIE
 from .mtv import (
@@ -578,7 +576,11 @@ from .tmz import (
     TMZIE,
     TMZArticleIE,
 )
-from .tnaflix import TNAFlixIE
+from .tnaflix import (
+    TNAFlixIE,
+    EMPFlixIE,
+    MovieFapIE,
+)
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
deleted file mode 100644
index 4827022e0..000000000
--- a/youtube_dl/extractor/empflix.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from __future__ import unicode_literals
-
-from .tnaflix import TNAFlixIE
-
-
-class EMPFlixIE(TNAFlixIE):
-    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
-
-    _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
-    _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
-    _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-
-    _TESTS = [
-        {
-            'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-            'md5': 'b1bc15b6412d33902d6e5952035fcabc',
-            'info_dict': {
-                'id': '33051',
-                'display_id': 'Amateur-Finger-Fuck',
-                'ext': 'mp4',
-                'title': 'Amateur Finger Fuck',
-                'description': 'Amateur solo finger fucking.',
-                'thumbnail': 're:https?://.*\.jpg$',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
-            'only_matching': True,
-        }
-    ]
diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
deleted file mode 100644
index 82b863539..000000000
--- a/youtube_dl/extractor/moviefap.py
+++ /dev/null
@@ -1,135 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    str_to_int
-)
-from ..compat import compat_str
-
-
-class MovieFapIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
-    _TESTS = [{
-        # normal, multi-format video
-        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
-        'md5': '26624b4e2523051b550067d547615906',
-        'info_dict': {
-            'id': 'be9867c9416c19f54a4a',
-            'ext': 'mp4',
-            'title': 'Experienced MILF Amazing Handjob',
-            'description': 'Experienced MILF giving an Amazing Handjob',
-            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
-            'uploader_id': 'darvinfred06',
-            'display_id': 'experienced-milf-amazing-handjob',
-            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
-        }
-    }, {
-        # quirky single-format case where the extension is given as fid, but the video is really an flv
-        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
-        'md5': 'fa56683e291fc80635907168a743c9ad',
-        'info_dict': {
-            'id': 'e5da0d3edce5404418f5',
-            'ext': 'flv',
-            'title': 'Jeune Couple Russe',
-            'description': 'Amateur',
-            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
-            'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe',
-            'categories': ['Amateur', 'Teen']
-        }
-    }]
-
-    @staticmethod
-    def __get_thumbnail_data(xml):
-
-        """
-        Constructs a list of video thumbnails from timeline preview images.
-        :param xml: the information XML document to parse
-        """
-
-        timeline = xml.find('timeline')
-        if timeline is None:
-            # not all videos have the data - ah well
-            return []
-
-        # get the required information from the XML
-        width = str_to_int(timeline.find('imageWidth').text)
-        height = str_to_int(timeline.find('imageHeight').text)
-        first = str_to_int(timeline.find('imageFirst').text)
-        last = str_to_int(timeline.find('imageLast').text)
-        pattern = timeline.find('imagePattern').text
-
-        # generate the list of thumbnail information dicts
-        thumbnails = []
-        for i in range(first, last + 1):
-            thumbnails.append({
-                'url': pattern.replace('#', compat_str(i)),
-                'width': width,
-                'height': height
-            })
-        return thumbnails
-
-    def _real_extract(self, url):
-
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex(
-            r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
-        xml = self._download_xml(info_url, video_id)
-
-        # find the video container
-        if xml.find('videoConfig') is not None:
-            ext = xml.find('videoConfig').find('type').text
-        else:
-            ext = 'flv'  # guess...
-
-        # work out the video URL(s)
-        formats = []
-        if xml.find('videoLink') is not None:
-            # single format available
-            formats.append({
-                'url': xpath_text(xml, 'videoLink', 'url', True),
-                'ext': ext
-            })
-        else:
-            # multiple formats available
-            for item in xml.find('quality').findall('item'):
-                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
-                formats.append({
-                    'url': xpath_text(item, 'videoLink', 'url', True),
-                    'ext': ext,
-                    'resolution': resolution,
-                    'height': int(re.findall(r'\d+', resolution)[0])
-                })
-
-            self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': self._html_search_regex(
-                r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
-            'display_id': re.compile(self._VALID_URL).match(url).group('name'),
-            'thumbnails': self.__get_thumbnail_data(xml),
-            'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex(
-                r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex(
-                r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex(
-                r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex(
-                r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex(
-                r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
-            'age_limit': 18,
-            'webpage_url': self._html_search_regex(
-                r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex(
-                r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
-        }
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index c282865b2..49516abca 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -3,39 +3,70 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    parse_duration,
     fix_xml_ampersands,
+    float_or_none,
+    int_or_none,
+    parse_duration,
+    str_to_int,
+    xpath_text,
 )
 
 
-class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
-
-    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
-    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
-    _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-
-    _TESTS = [
-        {
-            'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-            'md5': 'ecf3498417d09216374fc5907f9c6ec0',
-            'info_dict': {
-                'id': '553878',
-                'display_id': 'Carmella-Decesare-striptease',
-                'ext': 'mp4',
-                'title': 'Carmella Decesare - striptease',
-                'description': '',
-                'thumbnail': 're:https?://.*\.jpg$',
-                'duration': 91,
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-            'only_matching': True,
-        }
+class TNAFlixNetworkBaseIE(InfoExtractor):
+    # May be overridden in descendants if necessary
+    _CONFIG_REGEX = [
+        r'flashvars\.config\s*=\s*escape\("([^"]+)"',
+        r'<input[^>]+name="config\d?" value="([^"]+)"',
     ]
+    _TITLE_REGEX = r'<input[^>]+name="title" value="([^"]+)"'
+    _DESCRIPTION_REGEX = r'<input[^>]+name="description" value="([^"]+)"'
+    _UPLOADER_REGEX = r'<input[^>]+name="username" value="([^"]+)"'
+    _VIEW_COUNT_REGEX = None
+    _COMMENT_COUNT_REGEX = None
+    _AVERAGE_RATING_REGEX = None
+    _CATEGORIES_REGEX = r'<li[^>]*>\s*<span[^>]+class="infoTitle"[^>]*>Categories:</span>\s*<span[^>]+class="listView"[^>]*>(.+?)</span>\s*</li>'
+
+    def _extract_thumbnails(self, flix_xml):
+
+        def get_child(elem, names):
+            for name in names:
+                child = elem.find(name)
+                if child is not None:
+                    return child
+
+        timeline = get_child(flix_xml, ['timeline', 'rolloverBarImage'])
+        if timeline is None:
+            return
+
+        pattern_el = get_child(timeline, ['imagePattern', 'pattern'])
+        if pattern_el is None or not pattern_el.text:
+            return
+
+        first_el = get_child(timeline, ['imageFirst', 'first'])
+        last_el = get_child(timeline, ['imageLast', 'last'])
+        if first_el is None or last_el is None:
+            return
+
+        first_text = first_el.text
+        last_text = last_el.text
+        if not first_text.isdigit() or not last_text.isdigit():
+            return
+
+        first = int(first_text)
+        last = int(last_text)
+        if first > last:
+            return
+
+        width = int_or_none(xpath_text(timeline, './imageWidth', 'thumbnail width'))
+        height = int_or_none(xpath_text(timeline, './imageHeight', 'thumbnail height'))
+
+        return [{
+            'url': self._proto_relative_url(pattern_el.text.replace('#', compat_str(i)), 'http:'),
+            'width': width,
+            'height': height,
+        } for i in range(first, last + 1)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -44,39 +75,64 @@ class TNAFlixIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        cfg_url = self._proto_relative_url(self._html_search_regex(
+            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+
+        cfg_xml = self._download_xml(
+            cfg_url, display_id, 'Downloading metadata',
+            transform_source=fix_xml_ampersands)
+
+        formats = []
+
+        def extract_video_url(vl):
+            return re.sub('speed=\d+', 'speed=', vl.text)
+
+        video_link = cfg_xml.find('./videoLink')
+        if video_link is not None:
+            formats.append({
+                'url': extract_video_url(video_link),
+                'ext': xpath_text(cfg_xml, './videoConfig/type', 'type', default='flv'),
+            })
+
+        for item in cfg_xml.findall('./quality/item'):
+            video_link = item.find('./videoLink')
+            if video_link is None:
+                continue
+            res = item.find('res')
+            format_id = None if res is None else res.text
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]', format_id, 'height', default=None))
+            formats.append({
+                'url': self._proto_relative_url(extract_video_url(video_link), 'http:'),
+                'format_id': format_id,
+                'height': height,
+            })
+
+        self._sort_formats(formats)
+
+        thumbnail = self._proto_relative_url(
+            xpath_text(cfg_xml, './startThumb', 'thumbnail'), 'http:')
+        thumbnails = self._extract_thumbnails(cfg_xml)
+
         title = self._html_search_regex(
             self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
-        description = self._html_search_regex(
-            self._DESCRIPTION_REGEX, webpage, 'description', fatal=False, default='')
 
         age_limit = self._rta_search(webpage)
 
         duration = parse_duration(self._html_search_meta(
             'duration', webpage, 'duration', default=None))
 
-        cfg_url = self._proto_relative_url(self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+        def extract_field(pattern, name):
+            return self._html_search_regex(pattern, webpage, name, default=None) if pattern else None
 
-        cfg_xml = self._download_xml(
-            cfg_url, display_id, note='Downloading metadata',
-            transform_source=fix_xml_ampersands)
+        description = extract_field(self._DESCRIPTION_REGEX, 'description')
+        uploader = extract_field(self._UPLOADER_REGEX, 'uploader')
+        view_count = str_to_int(extract_field(self._VIEW_COUNT_REGEX, 'view count'))
+        comment_count = str_to_int(extract_field(self._COMMENT_COUNT_REGEX, 'comment count'))
+        average_rating = float_or_none(extract_field(self._AVERAGE_RATING_REGEX, 'average rating'))
 
-        thumbnail = self._proto_relative_url(
-            cfg_xml.find('./startThumb').text, 'http:')
-
-        formats = []
-        for item in cfg_xml.findall('./quality/item'):
-            video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
-            format_id = item.find('res').text
-            fmt = {
-                'url': self._proto_relative_url(video_url, 'http:'),
-                'format_id': format_id,
-            }
-            m = re.search(r'^(\d+)', format_id)
-            if m:
-                fmt['height'] = int(m.group(1))
-            formats.append(fmt)
-        self._sort_formats(formats)
+        categories_str = extract_field(self._CATEGORIES_REGEX, 'categories')
+        categories = categories_str.split(', ') if categories_str is not None else []
 
         return {
             'id': video_id,
@@ -84,7 +140,130 @@ class TNAFlixIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'duration': duration,
             'age_limit': age_limit,
+            'uploader': uploader,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'average_rating': average_rating,
+            'categories': categories,
             'formats': formats,
         }
+
+
+class TNAFlixIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
+
+    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
+    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
+    _UPLOADER_REGEX = r'(?s)<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)<div'
+
+    _TESTS = [{
+        # anonymous uploader, no categories
+        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'info_dict': {
+            'id': '553878',
+            'display_id': 'Carmella-Decesare-striptease',
+            'ext': 'mp4',
+            'title': 'Carmella Decesare - striptease',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 91,
+            'age_limit': 18,
+            'uploader': 'Anonymous',
+            'categories': [],
+        }
+    }, {
+        # non-anonymous uploader, categories
+        'url': 'https://www.tnaflix.com/teen-porn/Educational-xxx-video/video6538',
+        'md5': '0f5d4d490dbfd117b8607054248a07c0',
+        'info_dict': {
+            'id': '6538',
+            'display_id': 'Educational-xxx-video',
+            'ext': 'mp4',
+            'title': 'Educational xxx video',
+            'description': 'md5:b4fab8f88a8621c8fabd361a173fe5b8',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 164,
+            'age_limit': 18,
+            'uploader': 'bobwhite39',
+            'categories': ['Amateur Porn', 'Squirting Videos', 'Teen Girls 18+'],
+        }
+    }, {
+        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+        'only_matching': True,
+    }]
+
+
+class EMPFlixIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
+
+    _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
+
+    _TESTS = [{
+        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
+        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+        'info_dict': {
+            'id': '33051',
+            'display_id': 'Amateur-Finger-Fuck',
+            'ext': 'mp4',
+            'title': 'Amateur Finger Fuck',
+            'description': 'Amateur solo finger fucking.',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 83,
+            'age_limit': 18,
+            'uploader': 'cwbike',
+            'categories': ['Amateur', 'Anal', 'Fisting', 'Home made', 'Solo'],
+        }
+    }, {
+        'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
+        'only_matching': True,
+    }]
+
+
+class MovieFapIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<display_id>[^/]+)\.html'
+
+    _VIEW_COUNT_REGEX = r'<br>Views\s*<strong>([\d,.]+)</strong>'
+    _COMMENT_COUNT_REGEX = r'<span[^>]+id="comCount"[^>]*>([\d,.]+)</span>'
+    _AVERAGE_RATING_REGEX = r'Current Rating\s*<br>\s*<strong>([\d.]+)</strong>'
+    _CATEGORIES_REGEX = r'(?s)<div[^>]+id="vid_info"[^>]*>\s*<div[^>]*>.+?</div>(.*?)<br>'
+
+    _TESTS = [{
+        # normal, multi-format video
+        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
+        'md5': '26624b4e2523051b550067d547615906',
+        'info_dict': {
+            'id': 'be9867c9416c19f54a4a',
+            'display_id': 'experienced-milf-amazing-handjob',
+            'ext': 'mp4',
+            'title': 'Experienced MILF Amazing Handjob',
+            'description': 'Experienced MILF giving an Amazing Handjob',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+            'uploader': 'darvinfred06',
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing'],
+        }
+    }, {
+        # quirky single-format case where the extension is given as fid, but the video is really an flv
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'display_id': 'jeune-couple-russe',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+            'uploader': 'whiskeyjar',
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Amateur', 'Teen'],
+        }
+    }]

From 507683780eb14d012d4430044dc402d7e08e36b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 23:08:05 +0600
Subject: [PATCH 0549/2145] Credit @gebn for moviefap

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 889d599a2..117b9c219 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -128,3 +128,4 @@ Ping O.
 Mister Hat
 Peter Ding
 jackyzy823
+George Brighton

From c93d53f5e307dab1b5d03cd3c621a68f40ef840d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 00:48:06 +0600
Subject: [PATCH 0550/2145] [youtube] Fix likes/dislike extraction

---
 youtube_dl/extractor/youtube.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d9240ff02..8b43e274b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -29,6 +29,7 @@ from ..utils import (
     get_element_by_id,
     int_or_none,
     orderedSet,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
     uppercase_escape,
@@ -1005,12 +1006,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 video_description = ''
 
         def _extract_count(count_name):
-            count = self._search_regex(
-                r'id="watch-%s"[^>]*>.*?([\d,]+)\s*</span>' % re.escape(count_name),
-                video_webpage, count_name, default=None)
-            if count is not None:
-                return int(count.replace(',', ''))
-            return None
+            return str_to_int(self._search_regex(
+                r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
+                % re.escape(count_name),
+                video_webpage, count_name, default=None))
+
         like_count = _extract_count('like')
         dislike_count = _extract_count('dislike')
 

From 541462379153c19656aa52cc5796dbf05de874ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 00:49:19 +0600
Subject: [PATCH 0551/2145] [extractor/common] Remove superfluous line

---
 youtube_dl/extractor/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7fa46d295..81623bfe3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -36,7 +36,6 @@ from ..utils import (
 )
 
 
-
 class InfoExtractor(object):
     """Information Extractor class.
 

From 67134eaba1a56cec4117000acb2fc9284c9cdd9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Jun 2015 22:08:29 +0200
Subject: [PATCH 0552/2145] [YoutubeDL] rework how the format spec is processed

The spec string is processed using 'tokenize.tokenize' to split it in words and operators, the filters are still processed using regular expressions.
This should make easier to allow grouping operators with parens.
---
 test/test_YoutubeDL.py  |  27 ++--
 youtube_dl/YoutubeDL.py | 298 +++++++++++++++++++++++++---------------
 youtube_dl/compat.py    |   5 +
 3 files changed, 209 insertions(+), 121 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index a13c09ef4..8f7aef512 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -229,21 +229,30 @@ class TestFormatSelection(unittest.TestCase):
             '141', '172', '140', '171', '139',
         ]
 
-        for f1id, f2id in zip(order, order[1:]):
-            f1 = YoutubeIE._formats[f1id].copy()
-            f1['format_id'] = f1id
-            f1['url'] = 'url:' + f1id
-            f2 = YoutubeIE._formats[f2id].copy()
-            f2['format_id'] = f2id
-            f2['url'] = 'url:' + f2id
+        def format_info(f_id):
+            info = YoutubeIE._formats[f_id].copy()
+            info['format_id'] = f_id
+            info['url'] = 'url:' + f_id
+            return info
+        formats_order = [format_info(f_id) for f_id in order]
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '137+141')
+        self.assertEqual(downloaded['ext'], 'mp4')
+
+        for f1, f2 in zip(formats_order, formats_order[1:]):
             info_dict = _make_result([f1, f2], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
             downloaded = ydl.downloaded_info_dicts[0]
-            self.assertEqual(downloaded['format_id'], f1id)
+            self.assertEqual(downloaded['format_id'], f1['format_id'])
 
             info_dict = _make_result([f2, f1], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
@@ -251,7 +260,7 @@ class TestFormatSelection(unittest.TestCase):
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
             downloaded = ydl.downloaded_info_dicts[0]
-            self.assertEqual(downloaded['format_id'], f1id)
+            self.assertEqual(downloaded['format_id'], f1['format_id'])
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ef0f71bad..17a5407b9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -21,6 +21,7 @@ import subprocess
 import socket
 import sys
 import time
+import tokenize
 import traceback
 
 if os.name == 'nt':
@@ -34,6 +35,7 @@ from .compat import (
     compat_http_client,
     compat_kwargs,
     compat_str,
+    compat_tokenize_tokenize,
     compat_urllib_error,
     compat_urllib_request,
 )
@@ -851,8 +853,8 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
-    def _apply_format_filter(self, format_spec, available_formats):
-        " Returns a tuple of the remaining format_spec and filtered formats "
+    def _build_format_filter(self, filter_spec):
+        " Returns a function to filter the formats according to the filter_spec "
 
         OPERATORS = {
             '<': operator.lt,
@@ -862,13 +864,13 @@ class YoutubeDL(object):
             '=': operator.eq,
             '!=': operator.ne,
         }
-        operator_rex = re.compile(r'''(?x)\s*\[
+        operator_rex = re.compile(r'''(?x)\s*
             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
-            \]$
+            $
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
-        m = operator_rex.search(format_spec)
+        m = operator_rex.search(filter_spec)
         if m:
             try:
                 comparison_value = int(m.group('value'))
@@ -879,7 +881,7 @@ class YoutubeDL(object):
                 if comparison_value is None:
                     raise ValueError(
                         'Invalid value %r in format specification %r' % (
-                            m.group('value'), format_spec))
+                            m.group('value'), filter_spec))
             op = OPERATORS[m.group('op')]
 
         if not m:
@@ -887,85 +889,201 @@ class YoutubeDL(object):
                 '=': operator.eq,
                 '!=': operator.ne,
             }
-            str_operator_rex = re.compile(r'''(?x)\s*\[
+            str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9_-]+)
-                \s*\]$
+                \s*$
                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
-            m = str_operator_rex.search(format_spec)
+            m = str_operator_rex.search(filter_spec)
             if m:
                 comparison_value = m.group('value')
                 op = STR_OPERATORS[m.group('op')]
 
         if not m:
-            raise ValueError('Invalid format specification %r' % format_spec)
+            raise ValueError('Invalid filter specification %r' % filter_spec)
 
         def _filter(f):
             actual_value = f.get(m.group('key'))
             if actual_value is None:
                 return m.group('none_inclusive')
             return op(actual_value, comparison_value)
-        new_formats = [f for f in available_formats if _filter(f)]
+        return _filter
 
-        new_format_spec = format_spec[:-len(m.group(0))]
-        if not new_format_spec:
-            new_format_spec = 'best'
+    def build_format_selector(self, format_spec):
+        def syntax_error(note, start):
+            message = (
+                'Invalid format specification: '
+                '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
+            return SyntaxError(message)
 
-        return (new_format_spec, new_formats)
+        PICKFIRST = 'PICKFIRST'
+        MERGE = 'MERGE'
+        SINGLE = 'SINGLE'
+        FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 
-    def select_format(self, format_spec, available_formats):
-        while format_spec.endswith(']'):
-            format_spec, available_formats = self._apply_format_filter(
-                format_spec, available_formats)
-        if not available_formats:
-            return None
+        def _parse_filter(tokens):
+            filter_parts = []
+            for type, string, start, _, _ in tokens:
+                if type == tokenize.OP and string == ']':
+                    return ''.join(filter_parts)
+                else:
+                    filter_parts.append(string)
 
-        if format_spec in ['best', 'worst', None]:
-            format_idx = 0 if format_spec == 'worst' else -1
-            audiovideo_formats = [
-                f for f in available_formats
-                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
-            if audiovideo_formats:
-                return audiovideo_formats[format_idx]
-            # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
-            elif (all(f.get('acodec') != 'none' for f in available_formats) or
-                  all(f.get('vcodec') != 'none' for f in available_formats)):
-                return available_formats[format_idx]
-        elif format_spec == 'bestaudio':
-            audio_formats = [
-                f for f in available_formats
-                if f.get('vcodec') == 'none']
-            if audio_formats:
-                return audio_formats[-1]
-        elif format_spec == 'worstaudio':
-            audio_formats = [
-                f for f in available_formats
-                if f.get('vcodec') == 'none']
-            if audio_formats:
-                return audio_formats[0]
-        elif format_spec == 'bestvideo':
-            video_formats = [
-                f for f in available_formats
-                if f.get('acodec') == 'none']
-            if video_formats:
-                return video_formats[-1]
-        elif format_spec == 'worstvideo':
-            video_formats = [
-                f for f in available_formats
-                if f.get('acodec') == 'none']
-            if video_formats:
-                return video_formats[0]
-        else:
-            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
-            if format_spec in extensions:
-                filter_f = lambda f: f['ext'] == format_spec
-            else:
-                filter_f = lambda f: f['format_id'] == format_spec
-            matches = list(filter(filter_f, available_formats))
-            if matches:
-                return matches[-1]
-        return None
+        def _parse_format_selection(tokens, endwith=[]):
+            selectors = []
+            current_selector = None
+            for type, string, start, _, _ in tokens:
+                # ENCODING is only defined in python 3.x
+                if type == getattr(tokenize, 'ENCODING', None):
+                    continue
+                elif type in [tokenize.NAME, tokenize.NUMBER]:
+                    current_selector = FormatSelector(SINGLE, string, [])
+                elif type == tokenize.OP:
+                    if string in endwith:
+                        break
+                    if string == ',':
+                        selectors.append(current_selector)
+                        current_selector = None
+                    elif string == '/':
+                        first_choice = current_selector
+                        second_choice = _parse_format_selection(tokens, [','])
+                        current_selector = None
+                        selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
+                    elif string == '[':
+                        if not current_selector:
+                            current_selector = FormatSelector(SINGLE, 'best', [])
+                        format_filter = _parse_filter(tokens)
+                        current_selector.filters.append(format_filter)
+                    elif string == '+':
+                        video_selector = current_selector
+                        audio_selector = _parse_format_selection(tokens, [','])
+                        current_selector = None
+                        selectors.append(FormatSelector(MERGE, (video_selector, audio_selector), []))
+                    else:
+                        raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
+                elif type == tokenize.ENDMARKER:
+                    break
+            if current_selector:
+                selectors.append(current_selector)
+            return selectors
+
+        def _build_selector_function(selector):
+            if isinstance(selector, list):
+                fs = [_build_selector_function(s) for s in selector]
+
+                def selector_function(formats):
+                    for f in fs:
+                        for format in f(formats):
+                            yield format
+                return selector_function
+            elif selector.type == PICKFIRST:
+                fs = [_build_selector_function(s) for s in selector.selector]
+
+                def selector_function(formats):
+                    for f in fs:
+                        picked_formats = list(f(formats))
+                        if picked_formats:
+                            return picked_formats
+                    return []
+            elif selector.type == SINGLE:
+                format_spec = selector.selector
+
+                def selector_function(formats):
+                    if format_spec in ['best', 'worst', None]:
+                        format_idx = 0 if format_spec == 'worst' else -1
+                        audiovideo_formats = [
+                            f for f in formats
+                            if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+                        if audiovideo_formats:
+                            yield audiovideo_formats[format_idx]
+                        # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
+                        elif (all(f.get('acodec') != 'none' for f in formats) or
+                              all(f.get('vcodec') != 'none' for f in formats)):
+                            yield formats[format_idx]
+                    elif format_spec == 'bestaudio':
+                        audio_formats = [
+                            f for f in formats
+                            if f.get('vcodec') == 'none']
+                        if audio_formats:
+                            yield audio_formats[-1]
+                    elif format_spec == 'worstaudio':
+                        audio_formats = [
+                            f for f in formats
+                            if f.get('vcodec') == 'none']
+                        if audio_formats:
+                            yield audio_formats[0]
+                    elif format_spec == 'bestvideo':
+                        video_formats = [
+                            f for f in formats
+                            if f.get('acodec') == 'none']
+                        if video_formats:
+                            yield video_formats[-1]
+                    elif format_spec == 'worstvideo':
+                        video_formats = [
+                            f for f in formats
+                            if f.get('acodec') == 'none']
+                        if video_formats:
+                            yield video_formats[0]
+                    else:
+                        extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
+                        if format_spec in extensions:
+                            filter_f = lambda f: f['ext'] == format_spec
+                        else:
+                            filter_f = lambda f: f['format_id'] == format_spec
+                        matches = list(filter(filter_f, formats))
+                        if matches:
+                            yield matches[-1]
+            elif selector.type == MERGE:
+                def _merge(formats_info):
+                    format_1, format_2 = [f['format_id'] for f in formats_info]
+                    # The first format must contain the video and the
+                    # second the audio
+                    if formats_info[0].get('vcodec') == 'none':
+                        self.report_error('The first format must '
+                                          'contain the video, try using '
+                                          '"-f %s+%s"' % (format_2, format_1))
+                        return
+                    output_ext = (
+                        formats_info[0]['ext']
+                        if self.params.get('merge_output_format') is None
+                        else self.params['merge_output_format'])
+                    return {
+                        'requested_formats': formats_info,
+                        'format': '%s+%s' % (formats_info[0].get('format'),
+                                             formats_info[1].get('format')),
+                        'format_id': '%s+%s' % (formats_info[0].get('format_id'),
+                                                formats_info[1].get('format_id')),
+                        'width': formats_info[0].get('width'),
+                        'height': formats_info[0].get('height'),
+                        'resolution': formats_info[0].get('resolution'),
+                        'fps': formats_info[0].get('fps'),
+                        'vcodec': formats_info[0].get('vcodec'),
+                        'vbr': formats_info[0].get('vbr'),
+                        'stretched_ratio': formats_info[0].get('stretched_ratio'),
+                        'acodec': formats_info[1].get('acodec'),
+                        'abr': formats_info[1].get('abr'),
+                        'ext': output_ext,
+                    }
+                video_selector, audio_selector = map(_build_selector_function, selector.selector)
+
+                def selector_function(formats):
+                    formats = list(formats)
+                    for pair in itertools.product(video_selector(formats), audio_selector(formats)):
+                        yield _merge(pair)
+
+            filters = [self._build_format_filter(f) for f in selector.filters]
+
+            def final_selector(formats):
+                for _filter in filters:
+                    formats = list(filter(_filter, formats))
+                return selector_function(formats)
+            return final_selector
+
+        stream = io.BytesIO(format_spec.encode('utf-8'))
+        tokens = compat_tokenize_tokenize(stream.readline)
+        parsed_selector = _parse_format_selection(tokens)
+        return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):
         res = std_headers.copy()
@@ -1112,52 +1230,8 @@ class YoutubeDL(object):
         if req_format == 'all':
             formats_to_download = formats
         else:
-            for rfstr in req_format.split(','):
-                # We can accept formats requested in the format: 34/5/best, we pick
-                # the first that is available, starting from left
-                req_formats = rfstr.split('/')
-                for rf in req_formats:
-                    if re.match(r'.+?\+.+?', rf) is not None:
-                        # Two formats have been requested like '137+139'
-                        format_1, format_2 = rf.split('+')
-                        formats_info = (self.select_format(format_1, formats),
-                                        self.select_format(format_2, formats))
-                        if all(formats_info):
-                            # The first format must contain the video and the
-                            # second the audio
-                            if formats_info[0].get('vcodec') == 'none':
-                                self.report_error('The first format must '
-                                                  'contain the video, try using '
-                                                  '"-f %s+%s"' % (format_2, format_1))
-                                return
-                            output_ext = (
-                                formats_info[0]['ext']
-                                if self.params.get('merge_output_format') is None
-                                else self.params['merge_output_format'])
-                            selected_format = {
-                                'requested_formats': formats_info,
-                                'format': '%s+%s' % (formats_info[0].get('format'),
-                                                     formats_info[1].get('format')),
-                                'format_id': '%s+%s' % (formats_info[0].get('format_id'),
-                                                        formats_info[1].get('format_id')),
-                                'width': formats_info[0].get('width'),
-                                'height': formats_info[0].get('height'),
-                                'resolution': formats_info[0].get('resolution'),
-                                'fps': formats_info[0].get('fps'),
-                                'vcodec': formats_info[0].get('vcodec'),
-                                'vbr': formats_info[0].get('vbr'),
-                                'stretched_ratio': formats_info[0].get('stretched_ratio'),
-                                'acodec': formats_info[1].get('acodec'),
-                                'abr': formats_info[1].get('abr'),
-                                'ext': output_ext,
-                            }
-                        else:
-                            selected_format = None
-                    else:
-                        selected_format = self.select_format(rf, formats)
-                    if selected_format is not None:
-                        formats_to_download.append(selected_format)
-                        break
+            format_selector = self.build_format_selector(req_format)
+            formats_to_download = list(format_selector(formats))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f9529210d..bc218dd71 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -388,6 +388,10 @@ else:
             pass
         return _terminal_size(columns, lines)
 
+if sys.version_info >= (3, 0):
+    from tokenize import tokenize as compat_tokenize_tokenize
+else:
+    from tokenize import generate_tokens as compat_tokenize_tokenize
 
 __all__ = [
     'compat_HTTPError',
@@ -408,6 +412,7 @@ __all__ = [
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',
+    'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',

From 5acfa126c812c3ab7088af6c7df79697baee7831 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Jun 2015 22:48:02 +0200
Subject: [PATCH 0553/2145] [YoutubeDL] format spec: treat 'all' like a normal
 specifier

So you can use filters with it, for example 'all[width>=400][width<=600]'.
---
 test/test_YoutubeDL.py  |  5 +++++
 youtube_dl/YoutubeDL.py | 13 ++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8f7aef512..709e3100f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -317,6 +317,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+        ydl = YDL({'format': 'all[width>=400][width<=600]'})
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 17a5407b9..258e612af 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -990,7 +990,10 @@ class YoutubeDL(object):
                 format_spec = selector.selector
 
                 def selector_function(formats):
-                    if format_spec in ['best', 'worst', None]:
+                    if format_spec == 'all':
+                        for f in formats:
+                            yield f
+                    elif format_spec in ['best', 'worst', None]:
                         format_idx = 0 if format_spec == 'worst' else -1
                         audiovideo_formats = [
                             f for f in formats
@@ -1226,12 +1229,8 @@ class YoutubeDL(object):
                     req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
-        formats_to_download = []
-        if req_format == 'all':
-            formats_to_download = formats
-        else:
-            format_selector = self.build_format_selector(req_format)
-            formats_to_download = list(format_selector(formats))
+        format_selector = self.build_format_selector(req_format)
+        formats_to_download = list(format_selector(formats))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)

From c4bd188da46a837ddf8f8f8d4766eb799fa2b484 Mon Sep 17 00:00:00 2001
From: Anders Einar Hilden <hildenae@gmail.com>
Date: Mon, 29 Jun 2015 00:11:31 +0200
Subject: [PATCH 0554/2145] NRK now supports / requires HTTPS

Add s? to regexp to support new urls. Update testcases to use HTTPS.
---
 youtube_dl/extractor/nrk.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index cc70c2950..9e4581cf9 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class NRKIE(InfoExtractor):
-    _VALID_URL = r'(?:nrk:|http://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
+    _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
 
     _TESTS = [
         {
@@ -76,7 +76,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
@@ -116,11 +116,11 @@ class NRKPlaylistIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    _VALID_URL = r'(?P<baseurl>https?://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
-            'url': 'http://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
+            'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
             'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
@@ -132,7 +132,7 @@ class NRKTVIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://tv.nrk.no/program/mdfp15000514',
+            'url': 'https://tv.nrk.no/program/mdfp15000514',
             'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
@@ -145,7 +145,7 @@ class NRKTVIE(InfoExtractor):
         },
         {
             # single playlist video
-            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
+            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
             'md5': 'adbd1dbd813edaf532b0a253780719c2',
             'info_dict': {
                 'id': 'MSPO40010515-part2',
@@ -157,7 +157,7 @@ class NRKTVIE(InfoExtractor):
             'skip': 'Only works from Norway',
         },
         {
-            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
+            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
             'playlist': [
                 {
                     'md5': '9480285eff92d64f06e02a5367970a7a',

From bea41c7f3fa4f9072ad2f5354938ab1c8cef0a6d Mon Sep 17 00:00:00 2001
From: corone17 <reino@degeelebosch.nl>
Date: Mon, 29 Jun 2015 00:59:18 +0200
Subject: [PATCH 0555/2145] Update rtlnl.py

Better to extract 'http://manifest.us.rtl.nl' from the json, I'd say. And I think it's better to use the default json-url to make it more futureproof.
Succesfully tested with tarball.
---
 youtube_dl/extractor/rtlnl.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 41d202c28..e708e0093 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -51,7 +51,7 @@ class RtlNlIE(InfoExtractor):
     def _real_extract(self, url):
         uuid = self._match_id(url)
         info = self._download_json(
-            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=adaptive/' % uuid,
             uuid)
 
         material = info['material'][0]
@@ -60,8 +60,8 @@ class RtlNlIE(InfoExtractor):
         description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        videopath = material['videopath'].replace('.f4m', '.m3u8')
-        m3u8_url = 'http://manifest.us.rtl.nl' + videopath
+        videopath = material['videopath'].replace('adaptive', 'flash')
+        m3u8_url = info['meta']['videohost'] + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 

From 738b92632296a9fee3eb7e0c915f6ea6b395125f Mon Sep 17 00:00:00 2001
From: nawl <dylanlwrigh@gmail.com>
Date: Sun, 28 Jun 2015 17:24:00 -0600
Subject: [PATCH 0556/2145] [hentaistigma] Fix video extractor

---
 youtube_dl/extractor/hentaistigma.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index 63d87b74c..225af8cb3 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -32,7 +32,7 @@ class HentaiStigmaIE(InfoExtractor):
         wrap_webpage = self._download_webpage(wrap_url, video_id)
 
         video_url = self._html_search_regex(
-            r'clip:\s*{\s*url: "([^"]*)"', wrap_webpage, 'video url')
+            r'file:"([^"]+)"', wrap_webpage, 'video url')
 
         return {
             'id': video_id,

From 0130afb76e5cb6f470f39f127c8d09eea3e82d0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 29 Jun 2015 12:42:02 +0200
Subject: [PATCH 0557/2145] [YoutubeDL] format spec: allow grouping specifiers
 with parentheses

---
 test/test_YoutubeDL.py  | 24 ++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 39 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 709e3100f..6f374d7ea 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -245,6 +245,30 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], '137+141')
         self.assertEqual(downloaded['ext'], 'mp4')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['137+141', '248+141'])
+
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['136+141', '247+141'])
+
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['248+141'])
+
         for f1, f2 in zip(formats_order, formats_order[1:]):
             info_dict = _make_result([f1, f2], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 258e612af..e5b46f87e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -920,6 +920,7 @@ class YoutubeDL(object):
         PICKFIRST = 'PICKFIRST'
         MERGE = 'MERGE'
         SINGLE = 'SINGLE'
+        GROUP = 'GROUP'
         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 
         def _parse_filter(tokens):
@@ -942,6 +943,10 @@ class YoutubeDL(object):
                 elif type == tokenize.OP:
                     if string in endwith:
                         break
+                    elif string == ')':
+                        # ')' will be handled by the parentheses group
+                        tokens.restore_last_token()
+                        break
                     if string == ',':
                         selectors.append(current_selector)
                         current_selector = None
@@ -955,6 +960,10 @@ class YoutubeDL(object):
                             current_selector = FormatSelector(SINGLE, 'best', [])
                         format_filter = _parse_filter(tokens)
                         current_selector.filters.append(format_filter)
+                    elif string == '(':
+                        if current_selector:
+                            raise syntax_error('Unexpected "("', start)
+                        current_selector = FormatSelector(GROUP, _parse_format_selection(tokens, [')']), [])
                     elif string == '+':
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, [','])
@@ -977,6 +986,8 @@ class YoutubeDL(object):
                         for format in f(formats):
                             yield format
                 return selector_function
+            elif selector.type == GROUP:
+                selector_function = _build_selector_function(selector.selector)
             elif selector.type == PICKFIRST:
                 fs = [_build_selector_function(s) for s in selector.selector]
 
@@ -1084,8 +1095,32 @@ class YoutubeDL(object):
             return final_selector
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
-        tokens = compat_tokenize_tokenize(stream.readline)
-        parsed_selector = _parse_format_selection(tokens)
+        try:
+            tokens = list(compat_tokenize_tokenize(stream.readline))
+        except tokenize.TokenError:
+            raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
+
+        class TokenIterator(object):
+            def __init__(self, tokens):
+                self.tokens = tokens
+                self.counter = 0
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                if self.counter >= len(self.tokens):
+                    raise StopIteration()
+                value = self.tokens[self.counter]
+                self.counter += 1
+                return value
+
+            next = __next__
+
+            def restore_last_token(self):
+                self.counter -= 1
+
+        parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
         return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):

From cf386750c9194839e419a0412f45f25f28236c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 22:21:09 +0600
Subject: [PATCH 0558/2145] [hentaistigma] Modernize

---
 youtube_dl/extractor/hentaistigma.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index 225af8cb3..f5aa73d18 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -19,20 +17,19 @@ class HentaiStigmaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-            r'<h2 class="posttitle"><a[^>]*>([^<]+)</a>',
+            r'<h2[^>]+class="posttitle"[^>]*><a[^>]*>([^<]+)</a>',
             webpage, 'title')
         wrap_url = self._html_search_regex(
-            r'<iframe src="([^"]+mp4)"', webpage, 'wrapper url')
+            r'<iframe[^>]+src="([^"]+mp4)"', webpage, 'wrapper url')
         wrap_webpage = self._download_webpage(wrap_url, video_id)
 
         video_url = self._html_search_regex(
-            r'file:"([^"]+)"', wrap_webpage, 'video url')
+            r'file\s*:\s*"([^"]+)"', wrap_webpage, 'video url')
 
         return {
             'id': video_id,

From f0714c9f869a8b4c2032566af12c107491472361 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Jun 2015 01:02:48 +0600
Subject: [PATCH 0559/2145] [youtube] Speed up upload date regex (#6125)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c28ca9319..ed382e10b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1003,7 +1003,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
         if mobj is None:
             mobj = re.search(
-                r'(?s)id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live) on (.*?)</strong>',
+                r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.*?)</strong>',
                 video_webpage)
         if mobj is not None:
             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())

From c05724cb18eab295b4917a7e13f845e55cac56d9 Mon Sep 17 00:00:00 2001
From: alarig <alarig@swordarmor.fr>
Date: Tue, 30 Jun 2015 16:47:14 +0200
Subject: [PATCH 0560/2145] Add support of HTTPS for ina.fr

---
 youtube_dl/extractor/ina.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ina.py b/youtube_dl/extractor/ina.py
index 0847074ee..65712abc2 100644
--- a/youtube_dl/extractor/ina.py
+++ b/youtube_dl/extractor/ina.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class InaIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
     _TEST = {
         'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
         'md5': 'a667021bf2b41f8dc6049479d9bb38a3',

From 9d0b581feaa3adc72ed3a53b7e8c86fadfddb6e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Jun 2015 20:52:26 +0600
Subject: [PATCH 0561/2145] [youtube] Prefer meta for upload date and modernize

---
 youtube_dl/extractor/youtube.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ed382e10b..6769a009d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -999,15 +999,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
 
         # upload date
-        upload_date = None
-        mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
-        if mobj is None:
-            mobj = re.search(
-                r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.*?)</strong>',
-                video_webpage)
-        if mobj is not None:
-            upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
-            upload_date = unified_strdate(upload_date)
+        upload_date = self._html_search_meta(
+            'datePublished', video_webpage, 'upload date', default=None)
+        if not upload_date:
+            upload_date = self._search_regex(
+                [r'(?s)id="eow-date.*?>(.*?)</span>',
+                 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
+                video_webpage, 'upload date', default=None)
+            if upload_date:
+                upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
+        upload_date = unified_strdate(upload_date)
 
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',

From cf2ac6df6896dac4d23918867bb86fac1e1088d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 30 Jun 2015 19:45:42 +0200
Subject: [PATCH 0562/2145] [YoutubeDL] format spec: Fix handling of '+' with
 '/'

'bestvideo+bestaudio/best' was incorrectly interpreted as 'bestvideo+(bestaudio/best)', so it would fail if 'bestaudio' doesn't exist instead of falling back to 'best'.
---
 test/test_YoutubeDL.py  |  8 ++++++++
 youtube_dl/YoutubeDL.py | 25 +++++++++++++++----------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 6f374d7ea..1e4aaa559 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -245,6 +245,14 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], '137+141')
         self.assertEqual(downloaded['ext'], 'mp4')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '38')
+
         info_dict = _make_result(list(formats_order), extractor='youtube')
         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
         yie = YoutubeIE(ydl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5b46f87e..5deb4848e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -931,7 +931,7 @@ class YoutubeDL(object):
                 else:
                     filter_parts.append(string)
 
-        def _parse_format_selection(tokens, endwith=[]):
+        def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
             for type, string, start, _, _ in tokens:
@@ -941,18 +941,23 @@ class YoutubeDL(object):
                 elif type in [tokenize.NAME, tokenize.NUMBER]:
                     current_selector = FormatSelector(SINGLE, string, [])
                 elif type == tokenize.OP:
-                    if string in endwith:
+                    if string == ')':
+                        if not inside_group:
+                            # ')' will be handled by the parentheses group
+                            tokens.restore_last_token()
                         break
-                    elif string == ')':
-                        # ')' will be handled by the parentheses group
+                    elif inside_merge and string in ['/', ',']:
                         tokens.restore_last_token()
                         break
-                    if string == ',':
+                    elif inside_choice and string == ',':
+                        tokens.restore_last_token()
+                        break
+                    elif string == ',':
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
                         first_choice = current_selector
-                        second_choice = _parse_format_selection(tokens, [','])
+                        second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = None
                         selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
                     elif string == '[':
@@ -963,12 +968,12 @@ class YoutubeDL(object):
                     elif string == '(':
                         if current_selector:
                             raise syntax_error('Unexpected "("', start)
-                        current_selector = FormatSelector(GROUP, _parse_format_selection(tokens, [')']), [])
+                        group = _parse_format_selection(tokens, inside_group=True)
+                        current_selector = FormatSelector(GROUP, group, [])
                     elif string == '+':
                         video_selector = current_selector
-                        audio_selector = _parse_format_selection(tokens, [','])
-                        current_selector = None
-                        selectors.append(FormatSelector(MERGE, (video_selector, audio_selector), []))
+                        audio_selector = _parse_format_selection(tokens, inside_merge=True)
+                        current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                     else:
                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
                 elif type == tokenize.ENDMARKER:

From 1866432db74946c2b66263d38ed2c9d9d7e3177d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 30 Jun 2015 16:22:09 -0300
Subject: [PATCH 0563/2145] Rename --pp-params to --postprocessor-args and
 access value as super class attribute

---
 README.md                          |  2 +-
 youtube_dl/YoutubeDL.py            |  2 +-
 youtube_dl/__init__.py             |  6 +-----
 youtube_dl/options.py              |  4 ++--
 youtube_dl/postprocessor/common.py |  6 ++++--
 youtube_dl/postprocessor/ffmpeg.py | 11 +++++------
 6 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 813ac4a15..7eb17a163 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --pp-params                      Extra parameters for video post-processor.
+    --postprocessor-args             Extra parameters for video post-processor.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3bfe30c76..ff95add78 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -261,7 +261,7 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    pp_params:         Extra parameters for external apps, like avconv.
+    postprocessor_args: Extra parameters for external apps, like avconv.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 8b54d4ae2..356697015 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -171,10 +171,6 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
-    if opts.pp_params is None:
-        opts.pp_params = []
-    else:
-        opts.pp_params = shlex.split(opts.pp_params)
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
@@ -231,7 +227,7 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
-            'extra_params': opts.pp_params
+            'extra_cmd_args': opts.postprocessor_args,
         })
     if opts.convertsubtitles:
         postprocessors.append({
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index fbba9b9d8..3d88428c4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -688,8 +688,8 @@ def parseOpts(overrideArguments=None):
         metavar='FORMAT', dest='recodevideo', default=None,
         help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
     postproc.add_option(
-        '--pp-params',
-        dest='pp_params', default=None, metavar='ARGS',
+        '--postprocessor-args',
+        dest='postprocessor_args', default=None, metavar='ARGS',
         help='Extra parameters for video post-processor.')
     postproc.add_option(
         '-k', '--keep-video',
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index d944d9367..c44501b59 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import os
+import shlex
 
 from ..utils import (
     PostProcessingError,
@@ -23,12 +24,13 @@ class PostProcessor(object):
 
     PostProcessor objects follow a "mutual registration" process similar
     to InfoExtractor objects. And it can receive parameters from CLI trough
-    --pp-params.
+    --postprocessor-args.
     """
 
     _downloader = None
 
-    def __init__(self, downloader=None):
+    def __init__(self, downloader=None, extra_cmd_args=None):
+        self._extra_cmd_args = shlex.split(extra_cmd_args or '')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a696b12b4..891c72769 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -29,8 +29,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None):
-        PostProcessor.__init__(self, downloader)
+    def __init__(self, downloader=None, extra_cmd_args=None):
+        PostProcessor.__init__(self, downloader, extra_cmd_args)
         self._determine_executables()
 
     def check_version(self):
@@ -287,16 +287,15 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None, extra_params=[]):
-        super(FFmpegVideoConvertorPP, self).__init__(downloader)
+    def __init__(self, downloader=None, preferedformat=None, extra_cmd_args=None):
+        super(FFmpegVideoConvertorPP, self).__init__(downloader, extra_cmd_args)
         self._preferedformat = preferedformat
-        self._extra_params = extra_params
 
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
         ext = self._preferedformat
-        options = self._extra_params
+        options = self._extra_cmd_args
         if self._preferedformat == 'xvid':
             ext = 'avi'
             options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])

From 86511ea417ac22e02800fdc3efda1d73a87eb772 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Jul 2015 21:47:56 +0600
Subject: [PATCH 0564/2145] [drtuber] Fix extraction

---
 youtube_dl/extractor/drtuber.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 37c5c181f..639f9182c 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -36,25 +36,24 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            [r'class="hd_title" style="[^"]+">([^<]+)</h1>', r'<title>([^<]+) - \d+'],
+            [r'<p[^>]+class="title_substrate">([^<]+)</p>', r'<title>([^<]+) - \d+'],
             webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
-        like_count = str_to_int(self._html_search_regex(
-            r'<span id="rate_likes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
-            webpage, 'like count', fatal=False))
-        dislike_count = str_to_int(self._html_search_regex(
-            r'<span id="rate_dislikes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
-            webpage, 'like count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r'<span class="comments_count">([\d,\.]+)</span>',
-            webpage, 'comment count', fatal=False))
+        def extract_count(id_, name):
+            return str_to_int(self._html_search_regex(
+                r'<span[^>]+(?:class|id)="%s"[^>]*>([\d,\.]+)</span>' % id_,
+                webpage, '%s count' % name, fatal=False))
+
+        like_count = extract_count('rate_likes', 'like')
+        dislike_count = extract_count('rate_dislikes', 'dislike')
+        comment_count = extract_count('comments_count', 'comment')
 
         cats_str = self._search_regex(
-            r'<span>Categories:</span><div>(.+?)</div>', webpage, 'categories', fatal=False)
+            r'<div[^>]+class="categories_list">(.+?)</div>', webpage, 'categories', fatal=False)
         categories = [] if not cats_str else re.findall(r'<a title="([^"]+)"', cats_str)
 
         return {

From aa5d9a79d6b5c354ee4a6bfbb43f94c2485ab9b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Wed, 1 Jul 2015 20:12:26 -0300
Subject: [PATCH 0565/2145] Simplify `postprocessor_args` transmission to PP
 base class

* Remove `extra_cmd_args` transmission from sub to super class.
* Simplify params transmission through `downloader.params`.
---
 youtube_dl/__init__.py             | 2 +-
 youtube_dl/postprocessor/common.py | 5 ++---
 youtube_dl/postprocessor/ffmpeg.py | 8 ++++----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 356697015..249f76365 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -227,7 +227,6 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
-            'extra_cmd_args': opts.postprocessor_args,
         })
     if opts.convertsubtitles:
         postprocessors.append({
@@ -354,6 +353,7 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
+        'postprocessor_args': shlex.split(opts.postprocessor_args or ''),
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index c44501b59..bee64c457 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import os
-import shlex
 
 from ..utils import (
     PostProcessingError,
@@ -29,8 +28,8 @@ class PostProcessor(object):
 
     _downloader = None
 
-    def __init__(self, downloader=None, extra_cmd_args=None):
-        self._extra_cmd_args = shlex.split(extra_cmd_args or '')
+    def __init__(self, downloader=None):
+        self._extra_cmd_args = downloader.params.get('postprocessor_args')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 891c72769..de8c225da 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -29,8 +29,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None, extra_cmd_args=None):
-        PostProcessor.__init__(self, downloader, extra_cmd_args)
+    def __init__(self, downloader=None):
+        PostProcessor.__init__(self, downloader)
         self._determine_executables()
 
     def check_version(self):
@@ -287,8 +287,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None, extra_cmd_args=None):
-        super(FFmpegVideoConvertorPP, self).__init__(downloader, extra_cmd_args)
+    def __init__(self, downloader=None, preferedformat=None):
+        super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
 
     def run(self, information):

From dbe1a935269fe24fdbb894c325af4b8394c49c04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:05:43 +0800
Subject: [PATCH 0566/2145] [pbs] Fix player URL (closes #6139)

---
 youtube_dl/extractor/pbs.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 143a76696..289f930e0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -90,6 +90,21 @@ class PBSIE(InfoExtractor):
                 'id': 'united-states-of-secrets',
             },
             'playlist_count': 2,
+        },
+        {
+            'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
+            'info_dict': {
+                'id': '2280706814',
+                'display_id': 'player',
+                'ext': 'mp4',
+                'title': 'Death and the Civil War',
+                'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
+                'duration': 6705,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            }
         }
     ]
 
@@ -123,7 +138,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
+                r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 5b61070c700d2460e5aa8ea22f3aa806a2e17dc9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:08:48 +0800
Subject: [PATCH 0567/2145] [pbs] skip_download for m3u8 test cases

---
 youtube_dl/extractor/pbs.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 289f930e0..32c6adeba 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -35,6 +35,9 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
                 'duration': 3190,
             },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -46,6 +49,9 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
                 'duration': 5050,
             },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            }
         },
         {
             'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
@@ -68,7 +74,10 @@ class PBSIE(InfoExtractor):
                 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
-            }
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
@@ -82,7 +91,10 @@ class PBSIE(InfoExtractor):
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
-            }
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
@@ -104,7 +116,7 @@ class PBSIE(InfoExtractor):
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg
-            }
+            },
         }
     ]
 

From 93f942099351895c16ef558366f3eab241cd8793 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:13:27 +0800
Subject: [PATCH 0568/2145] [pbs] Add coding declaration

Python 2.x does not work without it.
---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 32c6adeba..1e2b965f9 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From 8d06a62485ddd2cff396287f6b39614afd1e6033 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 16:47:55 +0800
Subject: [PATCH 0569/2145] [npo] Decrypting token (closes #6136)

---
 youtube_dl/extractor/npo.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 5d8448571..41d7e3fcc 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -16,8 +16,23 @@ class NPOBaseIE(InfoExtractor):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
             video_id, note='Downloading token')
-        return self._search_regex(
+        token = self._search_regex(
             r'npoplayer\.token = "(.+?)"', token_page, 'token')
+        token_l = list(token)
+        first = second = None
+        for i in range(5, len(token_l) - 4):
+            if token_l[i].isdigit():
+                if first is None:
+                    first = i
+                elif second is None:
+                    second = i
+        if first is None or second is None:
+            first = 12
+            second = 13
+
+        token_l[first], token_l[second] = token_l[second], token_l[first]
+
+        return ''.join(token_l)
 
 
 class NPOIE(NPOBaseIE):
@@ -92,7 +107,7 @@ class NPOIE(NPOBaseIE):
 
     def _get_info(self, video_id):
         metadata = self._download_json(
-            'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
+            'http://e.omroep.nl/metadata/%s' % video_id,
             video_id,
             # We have to remove the javascript callback
             transform_source=strip_jsonp,

From f5ca97e393badab4efa07e358b051318955d0b6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jul 2015 20:20:09 +0600
Subject: [PATCH 0570/2145] [npo] Clarify token decryption algorithm source

---
 youtube_dl/extractor/npo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 41d7e3fcc..62d12b7a6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -18,6 +18,7 @@ class NPOBaseIE(InfoExtractor):
             video_id, note='Downloading token')
         token = self._search_regex(
             r'npoplayer\.token = "(.+?)"', token_page, 'token')
+        # Decryption algorithm extracted from http://npoplayer.omroep.nl/csjs/npoplayer-min.js
         token_l = list(token)
         first = second = None
         for i in range(5, len(token_l) - 4):

From 66e568de3b7ed73fd379b941f35ce2b3c7ad9664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jul 2015 21:39:46 +0600
Subject: [PATCH 0571/2145] [extractor/generic] Improve kaltura embeds support
 (Closes #6137)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 32e41d13e..ea60d4a96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -669,6 +669,18 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # Kaltura embed (different embed code)
+        {
+            'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
+            'info_dict': {
+                'id': '1_a52wc67y',
+                'ext': 'flv',
+                'upload_date': '20150127',
+                'uploader_id': 'PremierMedia',
+                'timestamp': int,
+                'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
+            },
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
@@ -1492,8 +1504,8 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = re.search(
-            r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
+        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
+                re.search(r'(?s)(["\'])(?:https?:)?//cdnapisec\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?\1.*?entry_id\s*:\s*(["\'])(?P<id>[^\2]+?)\2', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 

From 91b21b2334a3f8ba54a2b5c5fe4322c46565b9f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 3 Jul 2015 11:53:37 +0800
Subject: [PATCH 0572/2145] [infoq] Fix extraction (closes #6141)

---
 youtube_dl/extractor/infoq.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 117a7faf6..91a1b3ccb 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -5,6 +5,7 @@ import base64
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 
 
@@ -45,7 +46,7 @@ class InfoQIE(InfoExtractor):
         video_id, extension = video_filename.split('.')
 
         http_base = self._search_regex(
-            r'EXPRESSINSTALL_SWF\s*=\s*"(https?://[^/"]+/)', webpage,
+            r'EXPRESSINSTALL_SWF\s*=\s*[^"]*"((?:https?:)?//[^/"]+/)', webpage,
             'HTTP base URL')
 
         formats = [{
@@ -55,7 +56,7 @@ class InfoQIE(InfoExtractor):
             'play_path': playpath,
         }, {
             'format_id': 'http',
-            'url': http_base + real_id,
+            'url': compat_urlparse.urljoin(url, http_base) + real_id,
         }]
         self._sort_formats(formats)
 

From ddf0f74de77cba1d13445996725bfae4a66ac283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jul 2015 22:32:56 +0600
Subject: [PATCH 0573/2145] [howcast] Fix extraction and modernize

---
 youtube_dl/extractor/howcast.py | 35 +++++++++++++++++----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 3f7d6666c..16677f179 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import parse_iso8601
 
 
 class HowcastIE(InfoExtractor):
@@ -13,29 +12,31 @@ class HowcastIE(InfoExtractor):
         'info_dict': {
             'id': '390161',
             'ext': 'mp4',
-            'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.',
             'title': 'How to Tie a Square Knot Properly',
-        }
+            'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
+            'timestamp': 1276081287,
+            'upload_date': '20100609',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id = self._match_id(url)
 
-        video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
-        video_url = self._search_regex(r'\'?file\'?: "(http://mobile-media\.howcast\.com/[0-9]+\.mp4)',
-                                       webpage, 'video URL')
-
-        video_description = self._html_search_regex(r'<meta content=(?:"([^"]+)"|\'([^\']+)\') name=\'description\'',
-                                                    webpage, 'description', fatal=False)
+        embed_code = self._search_regex(
+            r'<iframe[^>]+src="[^"]+\bembed_code=([^\b]+)\b',
+            webpage, 'ooyala embed code')
 
         return {
+            '_type': 'url_transparent',
+            'ie_key': 'Ooyala',
+            'url': 'ooyala:%s' % embed_code,
             'id': video_id,
-            'url': video_url,
-            'title': self._og_search_title(webpage),
-            'description': video_description,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'timestamp': parse_iso8601(self._html_search_meta(
+                'article:published_time', webpage, 'timestamp')),
         }

From 0bcdc276530fe26725857eda5127aefc326ddc47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jul 2015 22:47:52 +0600
Subject: [PATCH 0574/2145] [dailymotion:cloud] Extend _VALID_URL (Closes
 #6145)

---
 youtube_dl/extractor/dailymotion.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 96f0ed9ad..8852f0add 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -254,22 +254,30 @@ class DailymotionUserIE(DailymotionPlaylistIE):
 
 
 class DailymotionCloudIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'http://api\.dmcloud\.net/embed/[^/]+/(?P<id>[^/?]+)'
+    _VALID_URL_PREFIX = r'http://api\.dmcloud\.net/(?:player/)?embed/'
+    _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
+    _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX
 
-    _TEST = {
+    _TESTS = [{
         # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
         # Tested at FranceTvInfo_2
         'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
         'only_matching': True,
-    }
+    }, {
+        # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html
+        'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_dmcloud_url(self, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage)
         if mobj:
             return mobj.group(1)
 
-        mobj = re.search(r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        mobj = re.search(
+            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL,
+            webpage)
         if mobj:
             return mobj.group(1)
 

From 2a46a27e6c704f7b99242243747fa7d8ac68cefc Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 01:07:48 -0700
Subject: [PATCH 0575/2145] [thisamericanlife] Add a new extractor

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/thisamericanlife.py | 32 ++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/thisamericanlife.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d44339200..aba62db53 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -569,6 +569,7 @@ from .tf1 import TF1IE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
+from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
new file mode 100644
index 000000000..6118afa1a
--- /dev/null
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -0,0 +1,32 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ThisAmericanLifeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
+        'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
+        'info_dict': {
+            'id': '487',
+            'title': '487: Harper High School, Part One',
+            'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
+            'ext': 'aac',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
+        media_url = 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': media_url,
+            'ext': 'aac',
+        }

From e41840c522fa09a08812c6e6724063f955426f21 Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 09:35:48 -0700
Subject: [PATCH 0576/2145] [thisamericanlife] get info from <meta> tags

---
 youtube_dl/extractor/thisamericanlife.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 6118afa1a..300884c1d 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 
 class ThisAmericanLifeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
+
     _TEST = {
         'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
         'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
@@ -14,19 +15,26 @@ class ThisAmericanLifeIE(InfoExtractor):
             'title': '487: Harper High School, Part One',
             'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
             'ext': 'aac',
-        }
+            'thumbnail': 'http://www.thisamericanlife.org/sites/default/files/imagecache/large_square/episodes/487_lg_2.jpg',
+            'description': 'We spent five months at Harper High School in Chicago, where last year alone 29 current and recent students were shot. 29. We went to get a sense of what it means to live in the midst of all this gun violence, how teens and adults navigate a world of funerals and Homecoming dances.',
+        },
+        'params': {
+            # m38u download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
-        media_url = 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8'
+        # TODO check to see if there's a free mp3. if so, download that, otherwise get the m3u8 stream.
 
         return {
             'id': video_id,
-            'title': title,
-            'url': media_url,
+            'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),
+            'url': 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8',
             'ext': 'aac',
+            'thumbnail': self._html_search_regex(r'<meta property="og:image" content="(.*?)"', webpage, 'thumbnail'),
+            'description': self._html_search_regex(r'<meta name="description" content="(.*?)"', webpage, 'description'),
         }

From 95506e37af27ee779dd667395f525e4a28455fc2 Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 09:50:39 -0700
Subject: [PATCH 0577/2145] [thisamericanlife] Remove unnecessary comment

---
 youtube_dl/extractor/thisamericanlife.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 300884c1d..295523271 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -28,8 +28,6 @@ class ThisAmericanLifeIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # TODO check to see if there's a free mp3. if so, download that, otherwise get the m3u8 stream.
-
         return {
             'id': video_id,
             'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),

From e56a4c9e9bc459346d23554207b910a1b772000d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jul 2015 05:42:53 +0600
Subject: [PATCH 0578/2145] [thisamericanlife] Improve and simplify

---
 youtube_dl/extractor/thisamericanlife.py | 42 +++++++++++++-----------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 295523271..36493a5de 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -1,38 +1,40 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
 
 class ThisAmericanLifeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
-
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/(?:radio-archives/episode/|play_full\.php\?play=)(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
-        'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
+        'md5': '8f7d2da8926298fdfca2ee37764c11ce',
         'info_dict': {
             'id': '487',
+            'ext': 'm4a',
             'title': '487: Harper High School, Part One',
-            'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
-            'ext': 'aac',
-            'thumbnail': 'http://www.thisamericanlife.org/sites/default/files/imagecache/large_square/episodes/487_lg_2.jpg',
-            'description': 'We spent five months at Harper High School in Chicago, where last year alone 29 current and recent students were shot. 29. We went to get a sense of what it means to live in the midst of all this gun violence, how teens and adults navigate a world of funerals and Homecoming dances.',
+            'description': 'md5:ee40bdf3fb96174a9027f76dbecea655',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # m38u download
-            'skip_download': True,
-        },
-    }
+    }, {
+        'url': 'http://www.thisamericanlife.org/play_full.php?play=487',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(
+            'http://www.thisamericanlife.org/radio-archives/episode/%s' % video_id, video_id)
 
         return {
             'id': video_id,
-            'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),
-            'url': 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8',
-            'ext': 'aac',
-            'thumbnail': self._html_search_regex(r'<meta property="og:image" content="(.*?)"', webpage, 'thumbnail'),
-            'description': self._html_search_regex(r'<meta name="description" content="(.*?)"', webpage, 'description'),
+            'url': 'http://stream.thisamericanlife.org/{0}/stream/{0}_64k.m3u8'.format(video_id),
+            'protocol': 'm3u8_native',
+            'ext': 'm4a',
+            'acodec': 'aac',
+            'vcodec': 'none',
+            'abr': 64,
+            'title': self._html_search_meta(r'twitter:title', webpage, 'title', fatal=True),
+            'description': self._html_search_meta(r'description', webpage, 'description'),
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From 35eb649e9d96027f4d3eec77841791524cc345e2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 4 Jul 2015 09:24:00 +0200
Subject: [PATCH 0579/2145] release 2015.07.04

---
 README.md              | 6 +++---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index e3452c9e1..93e7fb06f 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
@@ -190,8 +190,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
     -F, --list-formats               List all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
+    --youtube-skip-dash-manifest     Do not download the DASH manifests and related data on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv. Ignored if no
                                      merge is required
 
 ## Subtitle Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9a50fbd1c..687936103 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -283,6 +283,7 @@
  - **Motherless**
  - **Motorsport**: motorsport.com
  - **MovieClips**
+ - **MovieFap**
  - **Moviezine**
  - **movshare**: MovShare
  - **MPORA**
@@ -440,6 +441,8 @@
  - **smotri:broadcast**: Smotri.com broadcasts
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
+ - **SnagFilms**
+ - **SnagFilmsEmbed**
  - **Snotr**
  - **Sohu**
  - **soompi**
@@ -502,6 +505,7 @@
  - **TheOnion**
  - **ThePlatform**
  - **TheSixtyOne**
+ - **ThisAmericanLife**
  - **ThisAV**
  - **THVideo**
  - **THVideoPlaylist**
@@ -542,6 +546,7 @@
  - **twitch:stream**
  - **twitch:video**
  - **twitch:vod**
+ - **TwitterCard**
  - **Ubu**
  - **udemy**
  - **udemy:course**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a225e03a1..eff4aebeb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.25'
+__version__ = '2015.07.04'

From d7c9a3e976b00068e926e18d963e9fdb7c3cd678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jul 2015 17:22:11 +0600
Subject: [PATCH 0580/2145] Credit @remitamine for snagfilms (#6096)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 117b9c219..d5418dd37 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -129,3 +129,4 @@ Mister Hat
 Peter Ding
 jackyzy823
 George Brighton
+Remita Amine

From f5f4a27a964b41646303921104f4d6d6fd2098e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Jul 2015 21:30:26 +0200
Subject: [PATCH 0581/2145] [YoutubeDL] format spec: fix handling of '/' with
 ','

When using 'bestvideo/best,bestaudio', 'bestvideo/best' must be set as the current_selector (instead of appending it to the selectors), otherwise when it gets the ',' it would append 'None' to the selectors.
---
 test/test_YoutubeDL.py  | 8 ++++++++
 youtube_dl/YoutubeDL.py | 3 +--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 1e4aaa559..f103779d3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -253,6 +253,14 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], '38')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo/best,bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['137', '141'])
+
         info_dict = _make_result(list(formats_order), extractor='youtube')
         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
         yie = YoutubeIE(ydl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5deb4848e..5a79e5f1d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -958,8 +958,7 @@ class YoutubeDL(object):
                     elif string == '/':
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
-                        current_selector = None
-                        selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
+                        current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
                     elif string == '[':
                         if not current_selector:
                             current_selector = FormatSelector(SINGLE, 'best', [])

From bb8e55366289e0c129ef85abb8c1ac1cbae86a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Jul 2015 21:41:09 +0200
Subject: [PATCH 0582/2145] [YoutubeDL] format spec: Do not fail when a filter
 gives an empty result

For example with 'best[height<40]' we ended getting a 'IndexError: list index out of range'.
---
 test/test_YoutubeDL.py  | 9 ++++++++-
 youtube_dl/YoutubeDL.py | 3 +++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f103779d3..bf2baae07 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -15,7 +15,7 @@ from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_str
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
-from youtube_dl.utils import match_filter_func
+from youtube_dl.utils import ExtractorError, match_filter_func
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -362,6 +362,13 @@ class TestFormatSelection(unittest.TestCase):
         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
         self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
 
+        ydl = YDL({'format': 'best[height<40]'})
+        try:
+            ydl.process_ie_result(info_dict)
+        except ExtractorError:
+            pass
+        self.assertEqual(ydl.downloaded_info_dicts, [])
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a79e5f1d..6478d05dc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1005,6 +1005,9 @@ class YoutubeDL(object):
                 format_spec = selector.selector
 
                 def selector_function(formats):
+                    formats = list(formats)
+                    if not formats:
+                        return
                     if format_spec == 'all':
                         for f in formats:
                             yield f

From ede21449c8d87c2d000d16e5102ee85d87a9b14e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 06:29:36 +0600
Subject: [PATCH 0583/2145] [crunchyroll] Fix extraction (Closes #5855, closes
 #5881)

---
 youtube_dl/extractor/crunchyroll.py | 31 ++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 41f0c736d..73f1e22ef 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -27,7 +27,7 @@ from ..aes import (
 
 
 class CrunchyrollIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
@@ -45,6 +45,22 @@ class CrunchyrollIE(InfoExtractor):
             # rtmp
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
+        'info_dict': {
+            'id': '589804',
+            'ext': 'flv',
+            'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
+            'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Danny Choo Network',
+            'upload_date': '20120213',
+        },
+        'params': {
+            # rtmp
+            'skip_download': True,
+        },
+
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -251,16 +267,17 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
-            # urlencode doesn't work!
-            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' + stream_quality + '&media%5Fid=' + stream_id + '&video%5Fformat=' + stream_format
+            streamdata_req = compat_urllib_request.Request(
+                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
+                % (stream_id, stream_format, stream_quality),
+                compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
             streamdata = self._download_xml(
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
-            video_url = streamdata.find('./host').text
-            video_play_path = streamdata.find('./file').text
+            stream_info = streamdata.find('./{default}preload/stream_info')
+            video_url = stream_info.find('./host').text
+            video_play_path = stream_info.find('./file').text
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From 43232d5c143b1025671f70ff34f5c0f28ec56847 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 19:01:07 +0600
Subject: [PATCH 0584/2145] [rtlnl] Improve

---
 youtube_dl/extractor/rtlnl.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e708e0093..d94861325 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -59,9 +59,11 @@ class RtlNlIE(InfoExtractor):
         subtitle = material['title'] or info['episodes'][0]['name']
         description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
+        meta = info.get('meta', {})
+
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        videopath = material['videopath'].replace('adaptive', 'flash')
-        m3u8_url = info['meta']['videohost'] + videopath
+        videopath = material['videopath'].replace('/adaptive/', '/flash/')
+        m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
@@ -82,7 +84,7 @@ class RtlNlIE(InfoExtractor):
         self._sort_formats(formats)
 
         thumbnails = []
-        meta = info.get('meta', {})
+
         for p in ('poster_base_url', '"thumb_base_url"'):
             if not meta.get(p):
                 continue

From 9dfc4fa1a1a922fb6c0212d09d28b4a8495c1031 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 19:07:07 +0600
Subject: [PATCH 0585/2145] [rtlnl] Add test with encrypted m3u8 streams for
 reference

---
 youtube_dl/extractor/rtlnl.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index d94861325..049deaf8d 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -43,6 +43,10 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        # encrypted m3u8 streams
+        'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
+        'only_matching': True,
     }, {
         'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
         'only_matching': True,

From 89d42c2c758a3cb06b71145c24bfe8057419295a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jul 2015 02:58:02 +0600
Subject: [PATCH 0586/2145] [rtlnl] Clarify test

---
 youtube_dl/extractor/rtlnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 049deaf8d..1228245ad 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -44,7 +44,7 @@ class RtlNlIE(InfoExtractor):
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
     }, {
-        # encrypted m3u8 streams
+        # encrypted m3u8 streams, georestricted
         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
         'only_matching': True,
     }, {

From 0c20ee7d4b0c98221c71769482980a0a218dadb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jul 2015 04:16:56 +0600
Subject: [PATCH 0587/2145] [rtlnl] Clarify current adaptive -> flash
 workaround rationale

---
 youtube_dl/extractor/rtlnl.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 1228245ad..a4d3d73ff 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -66,6 +66,9 @@ class RtlNlIE(InfoExtractor):
         meta = info.get('meta', {})
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
+        # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
+        # this adaptive -> flash workaround is not required in general, but it also
+        # allows bypassing georestriction therefore is retained for now.
         videopath = material['videopath'].replace('/adaptive/', '/flash/')
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 

From 13af92fdc443f753dcb3cb91736b09a4ae85c36a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 08:39:38 +0800
Subject: [PATCH 0588/2145] [common] Add 'fatal' to _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 81623bfe3..d859aea52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,8 @@ class InfoExtractor(object):
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
-                              m3u8_id=None, note=None, errnote=None):
+                              m3u8_id=None, note=None, errnote=None,
+                              fatal=True):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -866,7 +867,10 @@ class InfoExtractor(object):
         m3u8_doc = self._download_webpage(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
-            errnote=errnote or 'Failed to download m3u8 information')
+            errnote=errnote or 'Failed to download m3u8 information',
+            fatal=fatal)
+        if m3u8_doc is False:
+            return m3u8_doc
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 59a83d3e5b9c09d0c6e8fb430ea99d71ef6e2eba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 08:40:38 +0800
Subject: [PATCH 0589/2145] [spiegeltv] Skip invalid m3u8 manifests (closes
 #6157)

---
 youtube_dl/extractor/spiegeltv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 08a5c4314..27f4033c5 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,11 +77,13 @@ class SpiegeltvIE(InfoExtractor):
                     'rtmp_live': True,
                 })
             elif determine_ext(endpoint) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     endpoint.replace('[video]', play_path),
                     video_id, 'm4v',
                     preference=1,  # Prefer hls since it allows to workaround georestriction
-                    m3u8_id='hls'))
+                    m3u8_id='hls', fatal=False)
+                if m3u8_formats is not False:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'url': endpoint,

From 37c1e4025c6df834e93a64c1c13eebac23e90942 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 6 Jul 2015 15:26:49 +0800
Subject: [PATCH 0590/2145] [yinyuetai] New extractor for yinyuetai.com

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/yinyuetai.py | 47 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/yinyuetai.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aba62db53..8665855eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -733,6 +733,7 @@ from .yandexmusic import (
     YandexMusicPlaylistIE,
 )
 from .yesjapan import YesJapanIE
+from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
new file mode 100644
index 000000000..661c34602
--- /dev/null
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class YinYueTaiIE(InfoExtractor):
+    IE_NAME = 'yinyuetai:video'
+    _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://v.yinyuetai.com/video/2322376',
+        'md5': '6e3abe28d38e3a54b591f9f040595ce0',
+        'info_dict': {
+            'id': '2322376',
+            'ext': 'mp4',
+            'title': '少女时代_PARTY_Music Video Teaser',
+            'creator': '少女时代',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://ext.yinyuetai.com/main/get-h-mv-info?json=true&videoId=%s' % video_id, video_id,
+            'Downloading mv info')['videoInfo']['coreVideoInfo']
+
+        if info['error']:
+            raise ExtractorError(info['errorMsg'], expected=True)
+
+        formats = [
+            {'url': format_info['videoUrl'], 'format_id': format_info['qualityLevel'],
+             'format': format_info['qualityLevelName'], 'filesize': format_info['fileSize'],
+             'ext': 'mp4', 'preference': format_info['bitrate']}
+            for format_info in info['videoUrlModels']
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['videoName'],
+            'thumbnail': info['bigHeadImage'],
+            'creator': info['artistNames'],
+            'duration': info['duration'],
+            'formats': formats,
+        }

From e9d33454b552c7a5a2e048f439b90a58aa5528ad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 16:19:49 +0800
Subject: [PATCH 0591/2145] [qqmusic:playlist] Playlist names are optional

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index f9aafcd28..d6bc05b7b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -274,6 +274,6 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
             ) for song in list_json['songlist']
         ]
 
-        list_name = list_json['dissname']
+        list_name = list_json.get('dissname')
         list_description = clean_html(unescapeHTML(list_json.get('desc')))
         return self.playlist_result(entries, list_id, list_name, list_description)

From fc7ae675e26a98bc74918f311d22f515ec7e0477 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:08:32 +0800
Subject: [PATCH 0592/2145] [qqmusic:album] Strip description

---
 youtube_dl/extractor/qqmusic.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e704640e5..6d85d58e3 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -169,7 +169,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
-            'description': 'md5:712f0cdbfc7e776820d08150e6df593d',
+            'description': 'md5:179c5dce203a5931970d306aa9607ea6',
         },
         'playlist_count': 4,
     }, {
@@ -177,7 +177,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '002Y5a3b3AlCu3',
             'title': '그리고...',
-            'description': 'md5:b1d133b8c9bac8fed4e1a97df759f4cf',
+            'description': 'md5:a48823755615508a95080e81b51ba729',
         },
         'playlist_count': 8,
     }]
@@ -196,6 +196,8 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         ]
         album_name = album['name']
         album_detail = album.get('desc')
+        if album_detail is not None:
+            album_detail = album_detail.strip()
 
         return self.playlist_result(entries, mid, album_name, album_detail)
 

From dfc4eca21f7d34b5e65f42b284b24077c8bbc109 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:09:17 +0800
Subject: [PATCH 0593/2145] [qqmusic:album] Playlist names are optional

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 6d85d58e3..03e6c688f 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -194,7 +194,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
                 'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
             ) for song in album['list']
         ]
-        album_name = album['name']
+        album_name = album.get('name')
         album_detail = album.get('desc')
         if album_detail is not None:
             album_detail = album_detail.strip()

From 85a064861f5213abf2b56dd671e1f45188a02adf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:54:41 +0800
Subject: [PATCH 0594/2145] [qqmusic] Use regex for thumbnails in test cases

---
 youtube_dl/extractor/qqmusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 8a724ab51..476432330 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -27,7 +27,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
-            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/7/p/001IV22P1RDX7p.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }, {
         'note': 'There is no mp3-320 version of this song.',
@@ -40,7 +40,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20050626',
             'creator': '李季美',
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
-            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/r/Q/0042owYj46IxrQ.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }]
 

From 275c0423aa85691fc78dda253fa5d00ab471e7b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jul 2015 00:02:34 +0600
Subject: [PATCH 0595/2145] [vk] Fix extraction (Closes #6153)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f2ae109f9..4ae28d4ca 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -153,7 +153,7 @@ class VKIE(InfoExtractor):
         if not video_id:
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
-        info_url = 'http://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+        info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
         ERRORS = {

From 7f220b2facecde21ac3257d20938b63c1b6b01fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jul 2015 00:04:19 +0600
Subject: [PATCH 0596/2145] [vk] Catch ownership confirmation request

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 4ae28d4ca..ccd897084 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -156,6 +156,11 @@ class VKIE(InfoExtractor):
         info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
+        if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
+            raise ExtractorError(
+                'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',
+                expected=True)
+
         ERRORS = {
             r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
             'Video %s has been removed from public access due to rightholder complaint.',

From d7b4d5dd5072fc82a7f3d437e5990aa74a35a100 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Jul 2015 14:16:56 +0800
Subject: [PATCH 0597/2145] [gfycat] Extract id correctly (fixes #6165)

---
 youtube_dl/extractor/gfycat.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 397f1d42e..048ee31e2 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class GfycatIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/)?(?P<id>[^/?#]+)'
+    _TESTS = [{
         'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
         'info_dict': {
             'id': 'DeadlyDecisiveGermanpinscher',
@@ -27,7 +27,23 @@ class GfycatIE(InfoExtractor):
             'categories': list,
             'age_limit': 0,
         }
-    }
+    }, {
+        'url': 'http://gfycat.com/ifr/JauntyTimelyAmazontreeboa',
+        'info_dict': {
+            'id': 'JauntyTimelyAmazontreeboa',
+            'ext': 'mp4',
+            'title': 'JauntyTimelyAmazontreeboa',
+            'timestamp': 1411720126,
+            'upload_date': '20140926',
+            'uploader': 'anonymous',
+            'duration': 3.52,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'categories': list,
+            'age_limit': 0,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 267dc07e6b71277b67abad08c515ca9c1bb09f61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Jul 2015 14:22:13 +0800
Subject: [PATCH 0598/2145] [gfycat] Catch errors

---
 youtube_dl/extractor/gfycat.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 048ee31e2..884700c52 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -6,6 +6,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     qualities,
+    ExtractorError,
 )
 
 
@@ -50,7 +51,10 @@ class GfycatIE(InfoExtractor):
 
         gfy = self._download_json(
             'http://gfycat.com/cajax/get/%s' % video_id,
-            video_id, 'Downloading video info')['gfyItem']
+            video_id, 'Downloading video info')
+        if 'error' in gfy:
+            raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)
+        gfy = gfy['gfyItem']
 
         title = gfy.get('title') or gfy['gfyName']
         description = gfy.get('description')

From 4dd09c9addeaccad6ad6d33f01d6240f3594fc84 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 7 Jul 2015 10:36:07 +0200
Subject: [PATCH 0599/2145] release 2015.07.07

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 687936103..0ca06c71d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -384,6 +384,7 @@
  - **Pyvideo**
  - **qqmusic**
  - **qqmusic:album**
+ - **qqmusic:playlist**
  - **qqmusic:singer**
  - **qqmusic:toplist**
  - **QuickVid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eff4aebeb..3364647ed 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.04'
+__version__ = '2015.07.07'

From cbc1fadd6f544619b711209b68ea15a912ca0fa1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jul 2015 13:40:21 +0800
Subject: [PATCH 0600/2145] [clipsyndicate] Support chic subdomain (fixes
 #6176)

---
 youtube_dl/extractor/clipsyndicate.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index d07d544ea..b1e45a677 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -10,9 +10,9 @@ from ..utils import (
 
 
 class ClipsyndicateIE(InfoExtractor):
-    _VALID_URL = r'http://www\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
         'md5': '4d7d549451bad625e0ff3d7bd56d776c',
         'info_dict': {
@@ -22,7 +22,10 @@ class ClipsyndicateIE(InfoExtractor):
             'duration': 612,
             'thumbnail': 're:^https?://.+\.jpg',
         },
-    }
+    }, {
+        'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 1316b54956ac871b4644b6760c5bf88207b2e58f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jul 2015 13:43:23 +0800
Subject: [PATCH 0601/2145] [clipsyndicate] Use _match_id

---
 youtube_dl/extractor/clipsyndicate.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index b1e45a677..8306d6fb7 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
@@ -28,8 +26,7 @@ class ClipsyndicateIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         js_player = self._download_webpage(
             'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id,
             video_id, 'Downlaoding player')

From cf9cf7dd04d8e15635084dcf9f7a76acff5ac218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:27:06 +0600
Subject: [PATCH 0602/2145] [vk] Extend _VALID_URL to handle biqle.ru (Closes
 #6179)

---
 youtube_dl/extractor/vk.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ccd897084..0f7ce45ca 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -21,7 +21,17 @@ from ..utils import (
 
 class VKIE(InfoExtractor):
     IE_NAME = 'vk.com'
-    _VALID_URL = r'https?://(?:m\.)?vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|(?:.+?\?.*?z=)?video(?P<videoid>[^s].*?)(?:\?|%2F|$))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:m\.)?vk\.com/video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|
+                            (?:
+                                (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
+                                (?:www\.)?biqle\.ru/watch/
+                            )
+                            (?P<videoid>[^s].*?)(?:\?|%2F|$)
+                        )
+                    '''
     _NETRC_MACHINE = 'vk'
 
     _TESTS = [
@@ -114,6 +124,11 @@ class VKIE(InfoExtractor):
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'only_matching': True,
         },
+        {
+            # vk wrapper
+            'url': 'http://www.biqle.ru/watch/847655_160197695',
+            'only_matching': True,
+        }
     ]
 
     def _login(self):

From 4647845679429739f72b0eba5b963e096ec5bf86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:34:50 +0600
Subject: [PATCH 0603/2145] [vk] Fix youtube extraction

---
 youtube_dl/extractor/vk.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0f7ce45ca..697064175 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -195,10 +195,11 @@ class VKIE(InfoExtractor):
             if re.search(error_re, info_page):
                 raise ExtractorError(error_msg % video_id, expected=True)
 
-        m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
-        if m_yt is not None:
-            self.to_screen('Youtube video detected')
-            return self.url_result(m_yt.group(1), 'Youtube')
+        youtube_url = self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+            info_page, 'youtube iframe', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube', video_id)
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)

From 9281f6d25338c179e7bc2f48352da3311102158e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:41:08 +0600
Subject: [PATCH 0604/2145] [vk] Add test for youtube embed

---
 youtube_dl/extractor/vk.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 697064175..23d153031 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -119,6 +119,21 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # youtube embed
+            'url': 'https://vk.com/video276849682_170681728',
+            'info_dict': {
+                'id': 'V3K4mi0SYkc',
+                'ext': 'mp4',
+                'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
+                'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
+                'duration': 179,
+                'upload_date': '20130116',
+                'uploader': "Children's Joy Foundation",
+                'uploader_id': 'thecjf',
+                'view_count': int,
+            },
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -199,7 +214,7 @@ class VKIE(InfoExtractor):
             r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
             info_page, 'youtube iframe', default=None)
         if youtube_url:
-            return self.url_result(youtube_url, 'Youtube', video_id)
+            return self.url_result(youtube_url, 'Youtube')
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)

From e2082ea9422aadf7ae2580d9333008279cda51f0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 00:50:32 +0800
Subject: [PATCH 0605/2145] [yinyuetai] Add test for h5/ part in _VALID_URL

---
 youtube_dl/extractor/yinyuetai.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 661c34602..41ee89da4 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
     _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',
         'md5': '6e3abe28d38e3a54b591f9f040595ce0',
         'info_dict': {
@@ -17,7 +17,10 @@ class YinYueTaiIE(InfoExtractor):
             'title': '少女时代_PARTY_Music Video Teaser',
             'creator': '少女时代',
         },
-    }
+    }, {
+        'url': 'http://v.yinyuetai.com/video/h5/2322376',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From af0f9b0e95233862e758140b282497d04edfb885 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 00:54:37 +0800
Subject: [PATCH 0606/2145] [yinyuetai] Style

---
 youtube_dl/extractor/yinyuetai.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 41ee89da4..a4ada4872 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -32,12 +32,14 @@ class YinYueTaiIE(InfoExtractor):
         if info['error']:
             raise ExtractorError(info['errorMsg'], expected=True)
 
-        formats = [
-            {'url': format_info['videoUrl'], 'format_id': format_info['qualityLevel'],
-             'format': format_info['qualityLevelName'], 'filesize': format_info['fileSize'],
-             'ext': 'mp4', 'preference': format_info['bitrate']}
-            for format_info in info['videoUrlModels']
-        ]
+        formats = [{
+            'url': format_info['videoUrl'],
+            'format_id': format_info['qualityLevel'],
+            'format': format_info['qualityLevelName'],
+            'filesize': format_info['fileSize'],
+            'ext': 'mp4',
+            'preference': format_info['bitrate'],
+        } for format_info in info['videoUrlModels']]
         self._sort_formats(formats)
 
         return {

From d76dea001b5365b7646986cba12f9908cd321f6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 01:07:45 +0800
Subject: [PATCH 0607/2145] [yinyuetai] Miscellaneous improvements

1. Include all fields in _TEST
2. Use .get() for optional fields
3. Clarify the intention of 'ext' in formats
---
 youtube_dl/extractor/yinyuetai.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index a4ada4872..003df9233 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -16,6 +16,8 @@ class YinYueTaiIE(InfoExtractor):
             'ext': 'mp4',
             'title': '少女时代_PARTY_Music Video Teaser',
             'creator': '少女时代',
+            'duration': 25,
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }, {
         'url': 'http://v.yinyuetai.com/video/h5/2322376',
@@ -35,18 +37,19 @@ class YinYueTaiIE(InfoExtractor):
         formats = [{
             'url': format_info['videoUrl'],
             'format_id': format_info['qualityLevel'],
-            'format': format_info['qualityLevelName'],
-            'filesize': format_info['fileSize'],
+            'format': format_info.get('qualityLevelName'),
+            'filesize': format_info.get('fileSize'),
+            # though URLs ends with .flv, the downloaded files are in fact mp4
             'ext': 'mp4',
-            'preference': format_info['bitrate'],
+            'tbr': format_info.get('bitrate'),
         } for format_info in info['videoUrlModels']]
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': info['videoName'],
-            'thumbnail': info['bigHeadImage'],
-            'creator': info['artistNames'],
-            'duration': info['duration'],
+            'thumbnail': info.get('bigHeadImage'),
+            'creator': info.get('artistNames'),
+            'duration': info.get('duration'),
             'formats': formats,
         }

From 082a0140eff05dfdac1020548a49541fcddd6ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 23:40:19 +0600
Subject: [PATCH 0608/2145] [yinyuetai] Do not capture unused group

---
 youtube_dl/extractor/yinyuetai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 003df9233..fa6b40816 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
-    _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',
         'md5': '6e3abe28d38e3a54b591f9f040595ce0',

From 77c6fb5b24c477c9de022f252dc9958a4b3b8b42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 20:48:38 +0600
Subject: [PATCH 0609/2145] [youtube] Make further DASH manifests not fatal
 after succeeded one

---
 youtube_dl/extractor/youtube.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6769a009d..3c629d38a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -798,7 +798,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
     def _parse_dash_manifest(
-            self, video_id, dash_manifest_url, player_url, age_gate):
+            self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
         def decrypt_sig(mobj):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@@ -807,7 +807,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         dash_doc = self._download_xml(
             dash_manifest_url, video_id,
             note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest')
+            errnote='Could not download DASH manifest',
+            fatal=fatal)
+
+        if dash_doc is False:
+            return []
 
         formats = []
         for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
@@ -1161,14 +1165,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
+            dash_mpd_fatal = True
             for dash_manifest_url in dash_mpds:
                 dash_formats = {}
                 try:
                     for df in self._parse_dash_manifest(
-                            video_id, dash_manifest_url, player_url, age_gate):
+                            video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df
+                        # Additional DASH manifests may end up in HTTP Error 403 therefore
+                        # allow them to fail without bug report message if we already have
+                        # some DASH manifest succeeded. This is temporary workaround to reduce
+                        # burst of bug reports until we figure out the reason and whether it
+                        # can be fixed at all.
+                        dash_mpd_fatal = False
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)

From 675e9f22ea3f43fbade9bbd13ee5de3eb45c538f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 23:52:03 +0600
Subject: [PATCH 0610/2145] [vimple] Extract spruto player based extractor
 class

---
 youtube_dl/extractor/vimple.py | 46 +++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index aa3d6ddfd..92321d66e 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -4,7 +4,29 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class VimpleIE(InfoExtractor):
+class SprutoBaseIE(InfoExtractor):
+    def _extract_spruto(self, spruto, video_id):
+        playlist = spruto['playlist'][0]
+        title = playlist['title']
+        video_id = playlist.get('videoId') or video_id
+        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
+        duration = int_or_none(playlist.get('duration'))
+
+        formats = [{
+            'url': f['url'],
+        } for f in playlist['video']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class VimpleIE(SprutoBaseIE):
     IE_DESC = 'Vimple - one-click video hosting'
     _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
     _TESTS = [
@@ -30,25 +52,9 @@ class VimpleIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://player.vimple.ru/iframe/%s' % video_id, video_id)
 
-        playlist = self._parse_json(
+        spruto = self._parse_json(
             self._search_regex(
                 r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
-            video_id)['playlist'][0]
+            video_id)
 
-        title = playlist['title']
-        video_id = playlist.get('videoId') or video_id
-        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
-        duration = int_or_none(playlist.get('duration'))
-
-        formats = [{
-            'url': f['url'],
-        } for f in playlist['video']]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        return self._extract_spruto(spruto, video_id)

From 1c20ddc966a69a241027c2d9a132b9caf3d0ebde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 23:53:50 +0600
Subject: [PATCH 0611/2145] [myvi:embed] Add extractor (Closes #6167)

---
 youtube_dl/extractor/myvi.py | 44 ++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/myvi.py

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
new file mode 100644
index 000000000..9f4330f50
--- /dev/null
+++ b/youtube_dl/extractor/myvi.py
@@ -0,0 +1,44 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .vimple import SprutoBaseIE
+
+
+class MyviEmbedIE(SprutoBaseIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        myvi\.ru/player/
+                            (?:
+                                (?:
+                                    embed/html|
+                                    api/Video/Get
+                                )/|
+                                content/preloader\.swf\?.*\bid=
+                            )
+                            (?P<id>[\da-zA-Z_]+)
+                    '''
+    _TESTS = [{
+        'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
+        'md5': '571bbdfba9f9ed229dc6d34cc0f335bf',
+        'info_dict': {
+            'id': 'f16b2bbd-cde8-481c-a981-7cd48605df43',
+            'ext': 'mp4',
+            'title': 'хозяин жизни',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 25,
+        },
+    }, {
+        'url': 'http://myvi.ru/player/content/preloader.swf?id=oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wOYf1WFpPfc_bWTKGVf_Zafr0',
+        'only_matching': True,
+    }, {
+        'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        spruto = self._download_json(
+            'http://myvi.ru/player/api/Video/Get/%s?sig' % video_id, video_id)['sprutoData']
+
+        return self._extract_spruto(spruto, video_id)

From 83423254ccdf60cb8756aaf7900c929c1cf1a3ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:07:26 +0600
Subject: [PATCH 0612/2145] [myvi:embed] Extend _VALID_URL

---
 youtube_dl/extractor/myvi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index 9f4330f50..a14a5365b 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -7,7 +7,7 @@ from .vimple import SprutoBaseIE
 class MyviEmbedIE(SprutoBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        myvi\.ru/player/
+                        myvi\.(?:ru/player|tv)/
                             (?:
                                 (?:
                                     embed/html|
@@ -33,6 +33,9 @@ class MyviEmbedIE(SprutoBaseIE):
     }, {
         'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
         'only_matching': True,
+    }, {
+        'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e6c2d9ad29bcc4eaa0eed03d3852588b6c7a10c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:25:36 +0600
Subject: [PATCH 0613/2145] [extractor/generic:myvi] Add support for myvi
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 youtube_dl/extractor/myvi.py    | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ea60d4a96..f8d6a8c76 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,6 +37,7 @@ from .rutv import RUTVIE
 from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
+from .myvi import MyviEmbedIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
@@ -1425,6 +1426,11 @@ class GenericIE(InfoExtractor):
         if smotri_url:
             return self.url_result(smotri_url, 'Smotri')
 
+        # Look for embedded Myvi.ru player
+        myvi_url = MyviEmbedIE._extract_url(webpage)
+        if myvi_url:
+            return self.url_result(myvi_url)
+
         # Look for embeded soundcloud player
         mobj = re.search(
             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index a14a5365b..896080c1e 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .vimple import SprutoBaseIE
 
 
@@ -38,6 +40,13 @@ class MyviEmbedIE(SprutoBaseIE):
         'only_matching': True,
     }]
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/embed/html/[^"]+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From f2f89c762a13392e8876c5e9dff8b418c3912ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:27:02 +0600
Subject: [PATCH 0614/2145] [myvi:embed] Improve _VALID_URL

---
 youtube_dl/extractor/myvi.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index 896080c1e..cd80cc177 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -13,11 +13,12 @@ class MyviEmbedIE(SprutoBaseIE):
                             (?:
                                 (?:
                                     embed/html|
+                                    flash|
                                     api/Video/Get
                                 )/|
                                 content/preloader\.swf\?.*\bid=
                             )
-                            (?P<id>[\da-zA-Z_]+)
+                            (?P<id>[\da-zA-Z_-]+)
                     '''
     _TESTS = [{
         'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
@@ -38,12 +39,15 @@ class MyviEmbedIE(SprutoBaseIE):
     }, {
         'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
         'only_matching': True,
+    }, {
+        'url': 'http://myvi.ru/player/flash/ocp2qZrHI-eZnHKQBK4cZV60hslH8LALnk0uBfKsB-Q4WnY26SeGoYPi8HWHxu0O30',
+        'only_matching': True,
     }]
 
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/embed/html/[^"]+)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/(?:embed/html|flash)/[^"]+)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 6dd94d3a79353f8e694efaf2fa27f4bb40227aff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:27:44 +0600
Subject: [PATCH 0615/2145] [myvi:embed] Rename to myvi

---
 youtube_dl/extractor/__init__.py | 1 +
 youtube_dl/extractor/generic.py  | 4 ++--
 youtube_dl/extractor/myvi.py     | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a0e3b333d..a64b457bd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -323,6 +323,7 @@ from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
+from .myvi import MyviIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f8d6a8c76..7c604c554 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,7 +37,7 @@ from .rutv import RUTVIE
 from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
-from .myvi import MyviEmbedIE
+from .myvi import MyviIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
@@ -1427,7 +1427,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(smotri_url, 'Smotri')
 
         # Look for embedded Myvi.ru player
-        myvi_url = MyviEmbedIE._extract_url(webpage)
+        myvi_url = MyviIE._extract_url(webpage)
         if myvi_url:
             return self.url_result(myvi_url)
 
diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index cd80cc177..4c65be122 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -6,7 +6,7 @@ import re
 from .vimple import SprutoBaseIE
 
 
-class MyviEmbedIE(SprutoBaseIE):
+class MyviIE(SprutoBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         myvi\.(?:ru/player|tv)/

From 06a12933f3621c4b5b84346edc0d5bd570f1aac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:58:01 +0600
Subject: [PATCH 0616/2145] [pbs] Add support for subtitles (Closes #6184)

---
 youtube_dl/extractor/pbs.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 1e2b965f9..fec5d65ad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -224,6 +224,14 @@ class PBSIE(InfoExtractor):
             rating_str = rating_str.rpartition('-')[2]
         age_limit = US_RATINGS.get(rating_str)
 
+        subtitles = {}
+        closed_captions_url = info.get('closed_captions_url')
+        if closed_captions_url:
+            subtitles['en'] = [{
+                'ext': 'ttml',
+                'url': closed_captions_url,
+            }]
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -234,4 +242,5 @@ class PBSIE(InfoExtractor):
             'age_limit': age_limit,
             'upload_date': upload_date,
             'formats': formats,
+            'subtitles': subtitles,
         }

From bf20b9c5405b276e6de29b1b28b2e3ad2182e891 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 01:15:55 +0600
Subject: [PATCH 0617/2145] [extractor/generic] Add test for myvi embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7c604c554..392ad3648 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -339,6 +339,17 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # Myvi.ru embed
+        {
+            'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
+            'info_dict': {
+                'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
+                'ext': 'mp4',
+                'title': 'Ужастики, русский трейлер (2015)',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 153,
+            }
+        },
         # XHamster embed
         {
             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',

From 02b386f80abc16f5334e980c2fb729b0cc69a0d6 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 13:29:57 +0800
Subject: [PATCH 0618/2145] [neteasemusic] Changes after review

---
 youtube_dl/extractor/neteasemusic.py | 42 ++++++++++++++++++----------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index a70c65ca5..419f3ecc9 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from hashlib import md5
+from base64 import b64encode
 from datetime import datetime
 import itertools
 import re
@@ -20,14 +21,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(str(cls._NETEASE_SALT))
+        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
         string_bytes = bytearray(str(dfsid))
         salt_len = len(salt_bytes)
-        for i in xrange(len(string_bytes)):
+        for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
         m.update(string_bytes)
-        result = m.digest().encode('base64')[:-1]
+        result = b64encode(m.digest())
         return result.replace('/', '_').replace('+', '-')
 
     @classmethod
@@ -41,12 +42,11 @@ class NetEaseMusicBaseIE(InfoExtractor):
                 'url': 'http://m1.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
-                'ext': details['extension'],
-                'abr': details['bitrate'] / 1000,
-                'preference': details['bitrate'],
+                'ext': details.get('extension'),
+                'abr': details.get('bitrate', 0) / 1000,
                 'format_id': song_format,
-                'filesize': details['size'],
-                'asr': details['sr']
+                'filesize': details.get('size'),
+                'asr': details.get('sr')
             })
         return formats
 
@@ -98,6 +98,19 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20080211',
             'timestamp': 1202745600,
         },
+    }, {
+        'note': 'Has translated name.',
+        'url': 'http://music.163.com/#/song?id=22735043',
+        'info_dict': {
+            'id': '22735043',
+            'ext': 'mp3',
+            'title': '소원을 말해봐 (Genie)',
+            'creator': '少女时代',
+            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
+            'upload_date': '20100127',
+            'timestamp': 1264608000,
+            'alt_title': '说出愿望吧(Genie)',
+        }
     }]
 
     def _process_lyrics(self, lyrics_info):
@@ -109,10 +122,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
         lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
         original_ts_texts = re.findall(lyrics_expr, original)
-        translation_ts_dict = {
-            time_stamp: text for time_stamp, text in re.findall(lyrics_expr, translated)
-        }
-
+        translation_ts_dict = dict(
+            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
+        )
         lyrics = '\n'.join([
             '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
             for time_stamp, text in original_ts_texts
@@ -139,8 +151,8 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
         lyrics = self._process_lyrics(lyrics_info)
 
         alt_title = None
-        if info.get('alias'):
-            alt_title = '/'.join(info.get('alias'))
+        if info.get('transNames'):
+            alt_title = '/'.join(info.get('transNames'))
 
         return {
             'id': song_id,
@@ -294,7 +306,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             mv_id, 'Downloading mv info')['data']
 
         formats = [
-            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'preference': int(brs)}
+            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
             for brs, mv_url in info['brs'].items()
         ]
         self._sort_formats(formats)

From 2b0fa1f7dd8d158b69eec4d17b254b99e976bc5c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 15:09:12 +0800
Subject: [PATCH 0619/2145] [kuwo] Merge KuwoSingerMusicIE into KuwoSingerIE

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 82cd85c44..a348b3077 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -262,7 +262,6 @@ from .kuwo import (
     KuwoAlbumIE,
     KuwoChartIE,
     KuwoSingerIE,
-    KuwoSingerMusicIE,
     KuwoCategoryIE,
     KuwoMvIE,
 )

From 1633491bff3e393d7d095b7303f954dacce4f4da Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 15:19:07 +0800
Subject: [PATCH 0620/2145] [kuwo] Merge KuwoSingerMusicIE into KuwoSingerIE
 (missed kuwo.py)

---
 youtube_dl/extractor/kuwo.py | 53 ++++++++++++------------------------
 1 file changed, 17 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 6a96a1aa4..82d5f3f95 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -180,15 +180,22 @@ class KuwoChartIE(InfoExtractor):
 
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/$'
-    _TEST = {
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
             'id': 'bruno+mars',
             'title': 'Bruno Mars',
         },
         'playlist_count': 10,
-    }
+    }, {
+        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
+        'info_dict': {
+            'id': 'Ali',
+            'title': 'Ali',
+        },
+        'playlist_mincount': 95,
+    }]
 
     def _real_extract(self, url):
         singer_id = self._match_id(url)
@@ -197,54 +204,28 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'姓名：<span>(.+?)</span>', webpage, 'singer name')
+            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
+        )
 
-        entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>',
-                webpage, flags=re.DOTALL)
-        ]
-        return self.playlist_result(entries, singer_id, singer_name)
-
-
-class KuwoSingerMusicIE(InfoExtractor):
-    IE_NAME = 'kuwo:singermusic'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/music(_[0-9]+)?.htm'
-    _TEST = {
-        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
-        'info_dict': {
-            'id': 'Ali',
-            'title': 'Ali的热门歌曲',
-        },
-        'playlist_mincount': 95,
-    }
-
-    def _real_extract(self, url):
-        singer_id = self._match_id(url)
-
-        list_name = None
         entries = []
+        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
                 singer_id, note='Download song list page #%d' % page_num,
                 errnote='Unable to get song list page #%d' % page_num)
 
-            if list_name is None:
-                list_name = self._html_search_regex(
-                    r'<h1>([^<>]+)<span>', webpage, 'list name')
-
             entries.extend([
                 self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
                     r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
-            ])
-            if not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+            ][:10 if first_page_only else None])
+
+            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
                 break
 
-        return self.playlist_result(entries, singer_id, list_name)
+        return self.playlist_result(entries, singer_id, singer_name)
 
 
 class KuwoCategoryIE(InfoExtractor):

From 15830339eff9076b1c4ca2cb6322fc1ec55233a8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:30:23 +0800
Subject: [PATCH 0621/2145] [neteasemusic] PEP8

---
 youtube_dl/extractor/neteasemusic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 419f3ecc9..e73c77f89 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -52,7 +52,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def convert_milliseconds(cls, ms):
-        return int(round(ms/1000.0))
+        return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
         req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
@@ -146,7 +146,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
         self._sort_formats(formats)
 
         lyrics_info = self.query_api(
-            'song/lyric?id=%s&lv=-1&tv=-1' % song_id, 
+            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
             song_id, 'Downloading lyrics data')
         lyrics = self._process_lyrics(lyrics_info)
 
@@ -183,7 +183,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
         album_id = self._match_id(url)
 
         info = self.query_api(
-            'album/%s?id=%s' % (album_id, album_id), 
+            'album/%s?id=%s' % (album_id, album_id),
             album_id, 'Downloading album data')['album']
 
         name = info['name']
@@ -221,7 +221,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         singer_id = self._match_id(url)
 
         info = self.query_api(
-            'artist/%s?id=%s' % (singer_id, singer_id), 
+            'artist/%s?id=%s' % (singer_id, singer_id),
             singer_id, 'Downloading singer data')
 
         name = info['artist']['name']
@@ -264,7 +264,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         list_id = self._match_id(url)
 
         info = self.query_api(
-            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id, 
+            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
             list_id, 'Downloading playlist data')['result']
 
         name = info['name']
@@ -378,7 +378,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
                 self.to_screen(
                     'Downloading just the main audio %s because of --no-playlist'
                     % info['mainSong']['id'])
-            
+
             formats = self.extract_formats(info['mainSong'])
             self._sort_formats(formats)
 
@@ -429,7 +429,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         entries = []
         for offset in itertools.count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
-                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d' 
+                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),
                 dj_id, 'Downloading dj programs - %d' % offset)
 

From 397a8ea96e228f918f127313f837f7de7dd78aed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:43:38 +0800
Subject: [PATCH 0622/2145] [neteasemusic] Encoding fixes for Python 2.6 and
 3.x

---
 youtube_dl/extractor/neteasemusic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index e73c77f89..9e99ddbda 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -11,6 +11,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_str,
 )
 
 
@@ -21,14 +22,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
-        string_bytes = bytearray(str(dfsid))
+        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
+        string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
         salt_len = len(salt_bytes)
         for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
-        m.update(string_bytes)
-        result = b64encode(m.digest())
+        m.update(bytes(string_bytes))
+        result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
     @classmethod

From a0e060ac1e9871974c02fe7cb3e812a3cbe79e73 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:58:12 +0800
Subject: [PATCH 0623/2145] [compat] Add compat_itertools_count

'step' parameter is added in Python 2.7
---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f9529210d..c3783337a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -9,6 +9,7 @@ import shutil
 import socket
 import subprocess
 import sys
+import itertools
 
 
 try:
@@ -388,6 +389,15 @@ else:
             pass
         return _terminal_size(columns, lines)
 
+try:
+    itertools.count(start=0, step=1)
+    compat_itertools_count = itertools.count
+except TypeError:  # Python 2.6
+    def compat_itertools_count(start=0, step=1):
+        n = start
+        while True:
+            yield n
+            n += step
 
 __all__ = [
     'compat_HTTPError',
@@ -401,6 +411,7 @@ __all__ = [
     'compat_html_entities',
     'compat_http_client',
     'compat_http_server',
+    'compat_itertools_count',
     'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',

From 4eab60cbd27ea8ce31ae9c6bf6b3f6e0e469d3af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:59:12 +0800
Subject: [PATCH 0624/2145] [netease:djradio] Use compat_itertools_count

---
 youtube_dl/extractor/neteasemusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 9e99ddbda..bdfe7e63f 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from hashlib import md5
 from base64 import b64encode
 from datetime import datetime
-import itertools
 import re
 
 from .common import InfoExtractor
@@ -12,6 +11,7 @@ from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
     compat_str,
+    compat_itertools_count,
 )
 
 
@@ -428,7 +428,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         name = None
         desc = None
         entries = []
-        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+        for offset in compat_itertools_count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
                 'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),

From a34af8d0667d8f4ceba3380f808a6d563ca01d77 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 19:13:52 +0800
Subject: [PATCH 0625/2145] [kuwo] PEP8

---
 youtube_dl/extractor/kuwo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 82d5f3f95..9c62191b5 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -249,10 +249,10 @@ class KuwoCategoryIE(InfoExtractor):
 
         category_name = self._html_search_regex(
             r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
-        
+
         category_desc = re.sub(
             r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
-        
+
         jsonm = self._parse_json(self._html_search_regex(
             r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
 

From 9296e92e1cbb5788c18c9a09590ccbc0e3faec68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 20:55:53 +0600
Subject: [PATCH 0626/2145] [twitch] Fix login (Closes #6186)

---
 youtube_dl/extractor/twitch.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b56ee2959..b7a72a7bd 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -22,8 +22,8 @@ class TwitchBaseIE(InfoExtractor):
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
-    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
+    _LOGIN_URL = 'https://secure.twitch.tv/login'
+    _LOGIN_POST_URL = 'https://passport.twitch.tv/authorize'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -59,20 +59,14 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        authenticity_token = self._search_regex(
-            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
-            login_page, 'authenticity token')
+        login_form = dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            login_page))
 
-        login_form = {
-            'utf8': '✓'.encode('utf-8'),
-            'authenticity_token': authenticity_token,
-            'redirect_on_login': '',
-            'embed_form': 'false',
-            'mp_source_action': 'login-button',
-            'follow': '',
+        login_form.update({
             'login': username,
             'password': password,
-        }
+        })
 
         request = compat_urllib_request.Request(
             self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))

From 89a683ae745de77f96ac8e82c8ff16d2675fa398 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:15:09 +0600
Subject: [PATCH 0627/2145] [twitch] Fix error message regex

---
 youtube_dl/extractor/twitch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b7a72a7bd..f912d3825 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -74,11 +74,12 @@ class TwitchBaseIE(InfoExtractor):
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        m = re.search(
-            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
-        if m:
+        error_message = self._search_regex(
+            r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
+            response, 'error message', default=None)
+        if error_message:
             raise ExtractorError(
-                'Unable to login: %s' % m.group('msg').strip(), expected=True)
+                'Unable to login. Twitch said: %s' % error_message, expected=True)
 
     def _prefer_source(self, formats):
         try:

From 17b41a3337b5326fd2c8dec8aa1ab285225e1daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:16:42 +0600
Subject: [PATCH 0628/2145] [twitch] Show reset password request

---
 youtube_dl/extractor/twitch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f912d3825..df809ecfe 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -81,6 +81,9 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login. Twitch said: %s' % error_message, expected=True)
 
+        if '>Reset your password<' in response:
+            self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
+
     def _prefer_source(self, formats):
         try:
             source = next(f for f in formats if f['format_id'] == 'Source')

From a31e3e7dcb9d0471d90ec8562934a144d25d7132 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:23:07 +0800
Subject: [PATCH 0629/2145] [kuwo] Regular expression improvements

1. Prevent .+ and .*
2. Use [^>]+ instead of spaces for HTML tags
3. Remove unnecessary trailing parts
---
 youtube_dl/extractor/kuwo.py         | 47 +++++++++++++++-------------
 youtube_dl/extractor/neteasemusic.py |  2 +-
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 9c62191b5..1095a26e2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -9,6 +9,7 @@ from ..utils import (
     get_element_by_id,
     clean_html,
     ExtractorError,
+    remove_start,
 )
 
 
@@ -73,10 +74,10 @@ class KuwoIE(InfoExtractor):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'<h1 title="(.+?)">', webpage, 'song name')
+            r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
         singer_name = self._html_search_regex(
-            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
-            flags=re.DOTALL, default=None)
+            r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
+            webpage, 'singer name', default=None)
         lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -84,7 +85,7 @@ class KuwoIE(InfoExtractor):
         formats = self._get_formats(song_id)
 
         album_id = self._html_search_regex(
-            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
+            r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
             webpage, 'album id', default=None, fatal=False)
 
         publish_time = None
@@ -131,15 +132,16 @@ class KuwoAlbumIE(InfoExtractor):
             errnote='Unable to get album info')
 
         album_name = self._html_search_regex(
-            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
-            'album name', flags=re.DOTALL)
-        album_intro = clean_html(
-            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
+            r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
+            'album name')
+        album_intro = remove_start(
+            clean_html(get_element_by_id("intro", webpage)),
+            '%s简介：' % album_name)
 
         entries = [
             self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
-                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
+                r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
                 webpage)
         ]
         return self.playlist_result(entries, album_id, album_name, album_intro)
@@ -147,7 +149,7 @@ class KuwoAlbumIE(InfoExtractor):
 
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
@@ -165,15 +167,15 @@ class KuwoChartIE(InfoExtractor):
             errnote='Unable to get chart info')
 
         chart_name = self._html_search_regex(
-            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
+            r'<h1[^>]+class="unDis">([^<]+)</h1>', webpage, 'chart name')
 
         chart_desc = self._html_search_regex(
-            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
+            r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
             self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
-                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
+                r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
         ]
         return self.playlist_result(entries, chart_id, chart_name, chart_desc)
 
@@ -204,11 +206,11 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
+            r'<div class="title clearfix">\s*<h1>([^<]+)<span', webpage, 'singer name'
         )
 
         entries = []
-        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
+        first_page_only = False if re.search(r'/music(?:_[0-9]+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
@@ -218,11 +220,11 @@ class KuwoSingerIE(InfoExtractor):
             entries.extend([
                 self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
-                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                    r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
             ][:10 if first_page_only else None])
 
-            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+            if first_page_only or not re.search(r'<a[^>]+href="[^"]+">下一页</a>', webpage):
                 break
 
         return self.playlist_result(entries, singer_id, singer_name)
@@ -248,13 +250,14 @@ class KuwoCategoryIE(InfoExtractor):
             errnote='Unable to get category info')
 
         category_name = self._html_search_regex(
-            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
+            r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
 
-        category_desc = re.sub(
-            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
+        category_desc = remove_start(
+            get_element_by_id("intro", webpage).strip(),
+            '%s简介：' % category_name)
 
         jsonm = self._parse_json(self._html_search_regex(
-            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
+            r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
 
         entries = [
             self.url_result(
@@ -289,7 +292,7 @@ class KuwoMvIE(KuwoIE):
             errnote='Unable to get mv detail info: %s' % song_id)
 
         mobj = re.search(
-            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
+            r'<h1[^>]+title="(?P<song>[^"]+)">[^<]+<span[^>]+title="(?P<singer>[^"]+)"',
             webpage)
         if mobj:
             song_name = mobj.group('song')
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index bdfe7e63f..ee52efaee 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -229,7 +229,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         if info['artist']['trans']:
             name = '%s - %s' % (name, info['artist']['trans'])
         if info['artist']['alias']:
-            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
 
         entries = [
             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],

From c0bf5e1c4dc0a236402c946b0b1fd358f9e39a74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:40:21 +0600
Subject: [PATCH 0630/2145] [twitch] Fix non-ASCII logins/passwords on python 2

---
 youtube_dl/extractor/twitch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index df809ecfe..49535cd80 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -64,8 +64,8 @@ class TwitchBaseIE(InfoExtractor):
             login_page))
 
         login_form.update({
-            'login': username,
-            'password': password,
+            'login': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
         })
 
         request = compat_urllib_request.Request(

From a9684c0dbf3879478fd223ce7594d58be7dffa4f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:46:44 +0800
Subject: [PATCH 0631/2145] [kuwo] Add KuwoBaseIE

---
 youtube_dl/extractor/kuwo.py | 63 +++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 1095a26e2..928f7f62d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -13,32 +13,7 @@ from ..utils import (
 )
 
 
-class KuwoIE(InfoExtractor):
-    IE_NAME = 'kuwo:song'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
-    _TESTS = [{
-        'url': 'http://www.kuwo.cn/yinyue/635632/',
-        'info_dict': {
-            'id': '635632',
-            'ext': 'ape',
-            'title': '爱我别走',
-            'creator': '张震岳',
-            'upload_date': '20080122',
-            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
-        },
-    }, {
-        'url': 'http://www.kuwo.cn/yinyue/6446136/',
-        'info_dict': {
-            'id': '6446136',
-            'ext': 'mp3',
-            'title': '心',
-            'creator': 'IU',
-            'upload_date': '20150518',
-        },
-        'params': {
-            'format': 'mp3-320'
-        },
-    }]
+class KuwoBaseIE(InfoExtractor):
     _FORMATS = [
         {'format': 'ape', 'ext': 'ape', 'preference': 100},
         {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
@@ -67,6 +42,34 @@ class KuwoIE(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
+
+class KuwoIE(KuwoBaseIE):
+    IE_NAME = 'kuwo:song'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/yinyue/635632/',
+        'info_dict': {
+            'id': '635632',
+            'ext': 'ape',
+            'title': '爱我别走',
+            'creator': '张震岳',
+            'upload_date': '20080122',
+            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
+        },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/6446136/',
+        'info_dict': {
+            'id': '6446136',
+            'ext': 'mp3',
+            'title': '心',
+            'creator': 'IU',
+            'upload_date': '20150518',
+        },
+        'params': {
+            'format': 'mp3-320'
+        },
+    }]
+
     def _real_extract(self, url):
         song_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -268,10 +271,10 @@ class KuwoCategoryIE(InfoExtractor):
         return self.playlist_result(entries, category_id, category_name, category_desc)
 
 
-class KuwoMvIE(KuwoIE):
+class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
     _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
             'id': '6480076',
@@ -279,8 +282,8 @@ class KuwoMvIE(KuwoIE):
             'title': '我们家MV',
             'creator': '2PM',
         },
-    }]
-    _FORMATS = KuwoIE._FORMATS + [
+    }
+    _FORMATS = KuwoBaseIE._FORMATS + [
         {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
         {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
     ]

From cf2c5fda4f3e753cc64098e6a751cf1a220efae7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:48:48 +0800
Subject: [PATCH 0632/2145] [kuwo] Use single quotes

---
 youtube_dl/extractor/kuwo.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 928f7f62d..2b5321cc2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -27,9 +27,9 @@ class KuwoBaseIE(InfoExtractor):
         formats = []
         for file_format in self._FORMATS:
             song_url = self._download_webpage(
-                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
+                'http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url' %
                 (file_format['ext'], file_format.get('br', ''), song_id),
-                song_id, note="Download %s url info" % file_format["format"],
+                song_id, note='Download %s url info' % file_format['format'],
             )
             if song_url.startswith('http://') or song_url.startswith('https://'):
                 formats.append({
@@ -81,7 +81,7 @@ class KuwoIE(KuwoBaseIE):
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', default=None)
-        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
+        lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
 
@@ -94,7 +94,7 @@ class KuwoIE(KuwoBaseIE):
         publish_time = None
         if album_id is not None:
             album_info_page = self._download_webpage(
-                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
+                'http://www.kuwo.cn/album/%s/' % album_id, song_id,
                 note='Download album detail info',
                 errnote='Unable to get album detail info')
 
@@ -138,11 +138,11 @@ class KuwoAlbumIE(InfoExtractor):
             r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
             'album name')
         album_intro = remove_start(
-            clean_html(get_element_by_id("intro", webpage)),
+            clean_html(get_element_by_id('intro', webpage)),
             '%s简介：' % album_name)
 
         entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
                 r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
                 webpage)
@@ -176,7 +176,7 @@ class KuwoChartIE(InfoExtractor):
             r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
                 r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
         ]
@@ -221,7 +221,7 @@ class KuwoSingerIE(InfoExtractor):
                 errnote='Unable to get song list page #%d' % page_num)
 
             entries.extend([
-                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+                self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
                     r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
@@ -256,7 +256,7 @@ class KuwoCategoryIE(InfoExtractor):
             r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
 
         category_desc = remove_start(
-            get_element_by_id("intro", webpage).strip(),
+            get_element_by_id('intro', webpage).strip(),
             '%s简介：' % category_name)
 
         jsonm = self._parse_json(self._html_search_regex(
@@ -264,7 +264,7 @@ class KuwoCategoryIE(InfoExtractor):
 
         entries = [
             self.url_result(
-                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
+                'http://www.kuwo.cn/yinyue/%s/' % song['musicrid'],
                 'Kuwo', song['musicrid'])
             for song in jsonm['musiclist']
         ]
@@ -301,7 +301,7 @@ class KuwoMvIE(KuwoBaseIE):
             song_name = mobj.group('song')
             singer_name = mobj.group('singer')
         else:
-            raise ExtractorError("Unable to find song or singer names")
+            raise ExtractorError('Unable to find song or singer names')
 
         formats = self._get_formats(song_id)
 

From 27713812a093ddfb329cd9cead878f3d0bf629cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:49:09 +0600
Subject: [PATCH 0633/2145] [extractor/common] Add method for extracting form
 hidden input fields as dict

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d859aea52..82f5de2d8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -705,6 +705,12 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
+    @staticmethod
+    def _form_hidden_inputs(html):
+        return dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            html))
+
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')

From a0aab26a414e8424e852a361c1822739f6bc936c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:49:39 +0600
Subject: [PATCH 0634/2145] [twitch] Use `_form_hidden_inputs` when logging in

---
 youtube_dl/extractor/twitch.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 49535cd80..af2b798fb 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -59,9 +59,7 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        login_form = dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            login_page))
+        login_form = self._form_hidden_inputs(login_page)
 
         login_form.update({
             'login': username.encode('utf-8'),

From a5839317aa5b41e7bc99ecae7405459bbcbff8a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:52:03 +0600
Subject: [PATCH 0635/2145] [vk] Use `_form_hidden_inputs` when logging in

---
 youtube_dl/extractor/vk.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 23d153031..c0292095b 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -154,9 +154,7 @@ class VKIE(InfoExtractor):
         login_page = self._download_webpage(
             'https://vk.com', None, 'Downloading login page')
 
-        login_form = dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            login_page))
+        login_form = self._form_hidden_inputs(login_page)
 
         login_form.update({
             'email': username.encode('cp1251'),

From 2688176c776a2d80c6a4ba85f6e2d07dfd8dba24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:53:38 +0600
Subject: [PATCH 0636/2145] [vodlocker] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/vodlocker.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 1c0966a79..431f4e2e3 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -28,12 +28,7 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!

From d3b8908886a78dd63441754f1ac20b70cb29de56 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:53:48 +0800
Subject: [PATCH 0637/2145] [kuwo] Simpler calls to url_result()

---
 youtube_dl/extractor/kuwo.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 2b5321cc2..69afacac9 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -142,9 +142,8 @@ class KuwoAlbumIE(InfoExtractor):
             '%s简介：' % album_name)
 
         entries = [
-            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
+            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                r'<p[^>]+class="listen"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+/)"',
                 webpage)
         ]
         return self.playlist_result(entries, album_id, album_name, album_intro)
@@ -176,9 +175,8 @@ class KuwoChartIE(InfoExtractor):
             r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
-            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
+            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/"', webpage)
         ]
         return self.playlist_result(entries, chart_id, chart_name, chart_desc)
 
@@ -221,9 +219,8 @@ class KuwoSingerIE(InfoExtractor):
                 errnote='Unable to get song list page #%d' % page_num)
 
             entries.extend([
-                self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-                for song_id in re.findall(
-                    r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                    r'<p[^>]+class="m_name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/',
                     webpage)
             ][:10 if first_page_only else None])
 
@@ -263,9 +260,7 @@ class KuwoCategoryIE(InfoExtractor):
             r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
 
         entries = [
-            self.url_result(
-                'http://www.kuwo.cn/yinyue/%s/' % song['musicrid'],
-                'Kuwo', song['musicrid'])
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song['musicrid'], 'Kuwo')
             for song in jsonm['musiclist']
         ]
         return self.playlist_result(entries, category_id, category_name, category_desc)

From de195c23a6837d1526198320561feba74886b9df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:54:39 +0600
Subject: [PATCH 0638/2145] [vimeo] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/vimeo.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cae90205d..d63c03183 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -452,11 +452,7 @@ class VimeoChannelIE(InfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', login_form))
+        fields = self._form_hidden_inputs(login_form)
         token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password

From 3b16d803c9633b7f3653d3d6d8022ed8642a09d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:55:22 +0600
Subject: [PATCH 0639/2145] [shared] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/shared.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 9f3e944e7..7fb68bc2d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -35,8 +35,7 @@ class SharedIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
-        download_form = dict(re.findall(
-            r'<input type="hidden" name="([^"]+)" value="([^"]*)"', webpage))
+        download_form = self._form_hidden_inputs(webpage)
         request = compat_urllib_request.Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')

From b8070dbbd72fa24fb4c0b7973bf547825a0cfed3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:56:14 +0600
Subject: [PATCH 0640/2145] [promptfile] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/promptfile.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index f536e6e6c..81a63c7fc 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -35,10 +35,7 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        fields = dict(re.findall(r'''(?x)type="hidden"\s+
-            name="(.+?)"\s+
-            value="(.*?)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
         req = compat_urllib_request.Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')

From 46f0f500163f64ed8276466667d6733e036bfa2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:56:41 +0600
Subject: [PATCH 0641/2145] [primesharetv] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/primesharetv.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 01cc3d9ea..94c9fb2cb 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -31,12 +31,7 @@ class PrimeShareTVIE(InfoExtractor):
         if '>File not exist<' in webpage:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         headers = {
             'Referer': url,

From 9f01c1a803d06034d443882f57528eba342a9c94 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:56:51 +0800
Subject: [PATCH 0642/2145] [kuwo] Use \d instead of [0-9]

---
 youtube_dl/extractor/kuwo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 69afacac9..18bf66404 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -45,7 +45,7 @@ class KuwoBaseIE(InfoExtractor):
 
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -116,7 +116,7 @@ class KuwoIE(KuwoBaseIE):
 
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
-    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
@@ -157,7 +157,7 @@ class KuwoChartIE(InfoExtractor):
         'info_dict': {
             'id': '香港中文龙虎榜',
             'title': '香港中文龙虎榜',
-            'description': 're:[0-9]{4}第[0-9]{2}期',
+            'description': 're:\d{4}第\d{2}期',
         },
         'playlist_mincount': 10,
     }
@@ -211,7 +211,7 @@ class KuwoSingerIE(InfoExtractor):
         )
 
         entries = []
-        first_page_only = False if re.search(r'/music(?:_[0-9]+)?\.htm', url) else True
+        first_page_only = False if re.search(r'/music(?:_\d+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
@@ -232,7 +232,7 @@ class KuwoSingerIE(InfoExtractor):
 
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
         'info_dict': {
@@ -268,7 +268,7 @@ class KuwoCategoryIE(InfoExtractor):
 
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
-    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {

From 01b89d5682adaecfd319897683b2043cfc7344c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:57:29 +0600
Subject: [PATCH 0643/2145] [played] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/played.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 45716c75d..9fe1524f2 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -38,9 +38,7 @@ class PlayedIE(InfoExtractor):
         if m_error:
             raise ExtractorError(m_error.group('msg'), expected=True)
 
-        fields = re.findall(
-            r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
-        data = dict(fields)
+        data = self._form_hidden_inputs(orig_webpage)
 
         self._sleep(2, video_id)
 

From 8fa7e5817a37756e4135fcb9daba47fc543be36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:58:24 +0600
Subject: [PATCH 0644/2145] [hostingbulk] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/hostingbulk.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 704d0285d..63f579592 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -58,11 +58,7 @@ class HostingBulkIE(InfoExtractor):
             r'<img src="([^"]+)".+?class="pic"',
             webpage, 'thumbnail', fatal=False)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         request = compat_urllib_request.Request(url, urlencode_postdata(fields))
         request.add_header('Content-type', 'application/x-www-form-urlencoded')

From 1c0163a5cc1d6b8fa5c7b4ee39b28e6e47c8981c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:58:49 +0600
Subject: [PATCH 0645/2145] [gorillavid] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/gorillavid.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 6147596e4..aabf07a20 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -78,12 +78,7 @@ class GorillaVidIE(InfoExtractor):
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             countdown = int_or_none(self._search_regex(

From 094790d2c963ef7ee9f5861e0458174ddd15ed87 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:03:49 +0800
Subject: [PATCH 0646/2145] [kuwo:song] Give warnings for unavailable optional
 fields

---
 youtube_dl/extractor/kuwo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 18bf66404..a021f3cdf 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -80,7 +80,7 @@ class KuwoIE(KuwoBaseIE):
             r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
-            webpage, 'singer name', default=None)
+            webpage, 'singer name', fatal=False)
         lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -89,7 +89,7 @@ class KuwoIE(KuwoBaseIE):
 
         album_id = self._html_search_regex(
             r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
-            webpage, 'album id', default=None, fatal=False)
+            webpage, 'album id', fatal=False)
 
         publish_time = None
         if album_id is not None:
@@ -100,7 +100,7 @@ class KuwoIE(KuwoBaseIE):
 
             publish_time = self._html_search_regex(
                 r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
-                'publish time', default=None)
+                'publish time', fatal=False)
             if publish_time:
                 publish_time = publish_time.replace('-', '')
 

From 446e7645001d276e6036b9a81e7996edc93b360b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:49:59 +0800
Subject: [PATCH 0647/2145] [baidu] Add localized name

---
 youtube_dl/extractor/baidu.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 906895c1e..e37ee4440 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
+    IE_DESC = '百度视频'
     _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',

From e1ba152352e95fb4cb6f3a5025166627d9df9787 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:50:31 +0800
Subject: [PATCH 0648/2145] [ctsnews] Add localized name

---
 youtube_dl/extractor/ctsnews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 0226f8036..45049bf37 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -6,6 +6,7 @@ from ..utils import parse_iso8601, ExtractorError
 
 
 class CtsNewsIE(InfoExtractor):
+    IE_DESC = '華視新聞'
     # https connection failed (Connection reset)
     _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{

From 513cbdda935b9f319cb61bffcdb0107249eb6b78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:50:45 +0800
Subject: [PATCH 0649/2145] [douyutv] Add localized name

---
 youtube_dl/extractor/douyutv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 479430c51..373b3b4b4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -9,6 +9,7 @@ from ..compat import (compat_str, compat_basestring)
 
 
 class DouyuTVIE(InfoExtractor):
+    IE_DESC = '斗鱼'
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',

From 44c514eb9c03ddf495feeaf94495bee39cffd842 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:03 +0800
Subject: [PATCH 0650/2145] [iqiyi] Add localized name

---
 youtube_dl/extractor/iqiyi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 9106dd074..0f6707d7c 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -20,6 +20,7 @@ from ..utils import (
 
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
+    IE_DESC = '爱奇艺'
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
 

From 0f08d7f851d8e41e09326d5e55dd93038d3ec752 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:14 +0800
Subject: [PATCH 0651/2145] [kuwo] Add localized name

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a021f3cdf..43f616a5d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -45,6 +45,7 @@ class KuwoBaseIE(InfoExtractor):
 
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
+    IE_DESC = '酷我音乐'
     _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',

From 963d0ce7e342841378945a8cb58d3c30bca02c60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:26 +0800
Subject: [PATCH 0652/2145] [letv] Add localized name

---
 youtube_dl/extractor/letv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index da896caf1..ba2ae8085 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 
 class LetvIE(InfoExtractor):
+    IE_DESC = '乐视网'
     _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
 
     _TESTS = [{

From 6ce89aecc330fe4b812526da4e119e7587548e27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:35 +0800
Subject: [PATCH 0653/2145] [neteasemusic] Add localized name

---
 youtube_dl/extractor/neteasemusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index ee52efaee..8758a6f0b 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -63,6 +63,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
 class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:song'
+    IE_DESC = '网易云音乐'
     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',

From 9d16788ad928800f4b6c5682dd08d0814a98754b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:55 +0800
Subject: [PATCH 0654/2145] [nextmedia] Add localized name

---
 youtube_dl/extractor/nextmedia.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index d1b7cff4c..c10784f6b 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -6,6 +6,7 @@ from ..utils import parse_iso8601
 
 
 class NextMediaIE(InfoExtractor):
+    IE_DESC = '蘋果日報'
     _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
@@ -66,6 +67,7 @@ class NextMediaIE(InfoExtractor):
 
 
 class NextMediaActionNewsIE(NextMediaIE):
+    IE_DESC = '蘋果日報 - 動新聞'
     _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
     _TESTS = [{
         'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
@@ -90,6 +92,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 
 class AppleDailyIE(NextMediaIE):
+    IE_DESC = '臺灣蘋果日報'
     _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',

From a7ada46bd90666b9c351bb54bd63b7225307afed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:07 +0800
Subject: [PATCH 0655/2145] [qqmusic] Add localized name

---
 youtube_dl/extractor/qqmusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 476432330..4c05fded4 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -16,6 +16,7 @@ from ..compat import compat_urllib_request
 
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
+    IE_DESC = 'QQ音乐'
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',

From 9b15be97aa0e094cccaf13e181b51b4399347d4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:19 +0800
Subject: [PATCH 0656/2145] [udn] Add localized name

---
 youtube_dl/extractor/udn.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index c08428acf..2151f8338 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -11,6 +11,7 @@ from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
+    IE_DESC = '聯合影音'
     _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',

From 4fa5f402321c71d6b2d8a281b60b8a23041c0e8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:56 +0800
Subject: [PATCH 0657/2145] [xuite] Add localized name

---
 youtube_dl/extractor/xuite.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 81d885fdc..5aac8adb3 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class XuiteIE(InfoExtractor):
+    IE_DESC = '隨意窩Xuite影音'
     _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
     _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{

From e014ff015d5240bac9371e5ffe5342ce025a574f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:53:08 +0800
Subject: [PATCH 0658/2145] [yam] Add localized name

---
 youtube_dl/extractor/yam.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 9d851bae3..001ee17b6 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class YamIE(InfoExtractor):
+    IE_DESC = '蕃薯藤yam天空部落'
     _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
 
     _TESTS = [{

From b931fbe5aba8e68a822f88d91d4ec424bc6509d3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:57:59 +0800
Subject: [PATCH 0659/2145] [yinyuetai] Add localized name

---
 youtube_dl/extractor/yinyuetai.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index fa6b40816..834d860af 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -7,6 +7,7 @@ from ..utils import ExtractorError
 
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
+    IE_DESC = '音悦Tai'
     _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',

From 246995dbc8ac0eb61355966eba7c7559a9ab7b24 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:58:13 +0800
Subject: [PATCH 0660/2145] [youku] Add localized name

---
 youtube_dl/extractor/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ced3a10cd..78caeb8b3 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -15,6 +15,7 @@ from ..compat import (
 
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
+    IE_DESC = '优酷'
     _VALID_URL = r'''(?x)
         (?:
             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|

From edd66be5be584dfb04d740cdd663fbd6203e3730 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:04 +0800
Subject: [PATCH 0661/2145] [kuwo] Add more localized names

---
 youtube_dl/extractor/kuwo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 43f616a5d..1077846f2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -117,6 +117,7 @@ class KuwoIE(KuwoBaseIE):
 
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
+    IE_DESC = '酷我音乐 - 专辑'
     _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
@@ -152,6 +153,7 @@ class KuwoAlbumIE(InfoExtractor):
 
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
+    IE_DESC = '酷我音乐 - 排行榜'
     _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
@@ -184,6 +186,7 @@ class KuwoChartIE(InfoExtractor):
 
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
+    IE_DESC = '酷我音乐 - 歌手'
     _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
@@ -233,6 +236,7 @@ class KuwoSingerIE(InfoExtractor):
 
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
+    IE_DESC = '酷我音乐 - 分类'
     _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
@@ -269,6 +273,7 @@ class KuwoCategoryIE(InfoExtractor):
 
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
+    IE_DESC = '酷我音乐 - MV'
     _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',

From ed848087d53d09797eec932e84c8c36c010d04a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:18 +0800
Subject: [PATCH 0662/2145] [neteasemusic] Add more localized names

---
 youtube_dl/extractor/neteasemusic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 8758a6f0b..a8e0a64ed 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -171,6 +171,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:album'
+    IE_DESC = '网易云音乐 - 专辑'
     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/album?id=220780',
@@ -200,6 +201,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:singer'
+    IE_DESC = '网易云音乐 - 歌手'
     _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'note': 'Singer has aliases.',
@@ -242,6 +244,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicListIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:playlist'
+    IE_DESC = '网易云音乐 - 歌单'
     _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/playlist?id=79177352',
@@ -287,6 +290,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:mv'
+    IE_DESC = '网易云音乐 - MV'
     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/mv?id=415350',
@@ -327,6 +331,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:program'
+    IE_DESC = '网易云音乐 - 电台节目'
     _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/program?id=10109055',
@@ -411,6 +416,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:djradio'
+    IE_DESC = '网易云音乐 - 电台'
     _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/djradio?id=42',

From 181c4ccaaa186685d4a5fce6f338486182079b9e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:36 +0800
Subject: [PATCH 0663/2145] [qqmusic] Add more localized names

---
 youtube_dl/extractor/qqmusic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 4c05fded4..1654a641f 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -143,6 +143,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
+    IE_DESC = 'QQ音乐 - 歌手'
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -187,6 +188,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
+    IE_DESC = 'QQ音乐 - 专辑'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TESTS = [{
@@ -229,6 +231,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
+    IE_DESC = 'QQ音乐 - 排行榜'
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
@@ -282,6 +285,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 
 class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:playlist'
+    IE_DESC = 'QQ音乐 - 歌单'
     _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 86f2541695ba0280982e47e52c5cf26946d5d7c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:22:33 +0200
Subject: [PATCH 0664/2145] Remove unused 're' imports

---
 youtube_dl/extractor/primesharetv.py | 2 --
 youtube_dl/extractor/shared.py       | 1 -
 youtube_dl/extractor/vodlocker.py    | 2 --
 3 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 94c9fb2cb..9aa0c862a 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 7fb68bc2d..6e2b94e7d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 431f4e2e3..4804692bf 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,

From 0a31a350981931ab9403d58258f5058ed98142a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:46:25 +0200
Subject: [PATCH 0665/2145] [YoutubeDL] format spec: add additional checks for
 invalid syntax

---
 test/test_YoutubeDL.py  | 10 ++++++++++
 youtube_dl/YoutubeDL.py |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bf2baae07..20f45f439 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -302,6 +302,16 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1['format_id'])
 
+    def test_invalid_format_specs(self):
+        def assert_syntax_error(format_spec):
+            ydl = YDL({'format': format_spec})
+            info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
+            self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
+
+        assert_syntax_error('bestvideo,,best')
+        assert_syntax_error('+bestaudio')
+        assert_syntax_error('bestvideo+')
+
     def test_format_filtering(self):
         formats = [
             {'format_id': 'A', 'filesize': 500, 'width': 1000},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6478d05dc..da7c51008 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -953,6 +953,8 @@ class YoutubeDL(object):
                         tokens.restore_last_token()
                         break
                     elif string == ',':
+                        if not current_selector:
+                            raise syntax_error('"," must follow a format selector', start)
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
@@ -972,6 +974,8 @@ class YoutubeDL(object):
                     elif string == '+':
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
+                        if not video_selector or not audio_selector:
+                            raise syntax_error('"+" must be between two format selectors', start)
                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                     else:
                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)

From dc48695ab9c84cda56ee723aee11e5da44c2258a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:59:45 +0200
Subject: [PATCH 0666/2145] Document how to group format selectors

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e3452c9e1..4e6cb3fc7 100644
--- a/README.md
+++ b/README.md
@@ -268,7 +268,7 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
 Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 

From fac54cb426d85572c6928ce0d9e5bf1efb459548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 04:43:29 +0600
Subject: [PATCH 0667/2145] [webofstories:playlist] Improve and add test

---
 youtube_dl/extractor/webofstories.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index d70e30c00..2037d9b3d 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -104,7 +104,14 @@ class WebOfStoriesIE(InfoExtractor):
 
 class WebOfStoriesPlaylistIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
-    _TESTS = []
+    _TEST = {
+        'url': 'http://www.webofstories.com/playAll/donald.knuth',
+        'info_dict': {
+            'id': 'donald.knuth',
+            'title': 'Donald Knuth (Scientist)',
+        },
+        'playlist_mincount': 97,
+    }
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -116,10 +123,19 @@ class WebOfStoriesPlaylistIE(InfoExtractor):
             for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
         ]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Web\sof\sStories</title>', webpage, 'title')
+        title = self._search_regex(
+            r'<div id="speakerName">\s*<span>([^<]+)</span>',
+            webpage, 'speaker', default=None)
+        if title:
+            field = self._search_regex(
+                r'<span id="primaryField">([^<]+)</span>',
+                webpage, 'field', default=None)
+            if field:
+                title += ' (%s)' % field
 
-        description = self._html_search_meta(
-            'description', webpage, 'description')
+        if not title:
+            title = self._search_regex(
+                r'<title>Play\s+all\s+stories\s*-\s*([^<]+)\s*-\s*Web\s+of\s+Stories</title>',
+                webpage, 'title')
 
-        return self.playlist_result(entries, playlist_id, title, description)
+        return self.playlist_result(entries, playlist_id, title)

From 79913fde35ead6247d7b764f3a95059fc4f17dbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 21:23:49 +0600
Subject: [PATCH 0668/2145] [vk] Add list id to info_url

---
 youtube_dl/extractor/vk.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c0292095b..333c2a634 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -29,7 +29,7 @@ class VKIE(InfoExtractor):
                                 (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
                                 (?:www\.)?biqle\.ru/watch/
                             )
-                            (?P<videoid>[^s].*?)(?:\?|%2F|$)
+                            (?P<videoid>[^s].*?)(?:\?(?:.*\blist=(?P<list_id>[\da-f]+))?|%2F|$)
                         )
                     '''
     _NETRC_MACHINE = 'vk'
@@ -119,6 +119,20 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # video (removed?) only available with list id
+            'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4',
+            'md5': '091287af5402239a1051c37ec7b92913',
+            'info_dict': {
+                'id': '171201961',
+                'ext': 'mp4',
+                'title': 'ТюменцевВВ_09.07.2015',
+                'uploader': 'Anton Ivanov',
+                'duration': 109,
+                'upload_date': '20150709',
+                'view_count': int,
+            },
+        },
         {
             # youtube embed
             'url': 'https://vk.com/video276849682_170681728',
@@ -182,6 +196,12 @@ class VKIE(InfoExtractor):
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
         info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+
+        # Some videos (removed?) can only be downloaded with list id specified
+        list_id = mobj.group('list_id')
+        if list_id:
+            info_url += '&list=%s' % list_id
+
         info_page = self._download_webpage(info_url, video_id)
 
         if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):

From d919fa3344463f94a152a10ba65981cd290d9ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 21:26:03 +0600
Subject: [PATCH 0669/2145] [vk] Handle access denied error

---
 youtube_dl/extractor/vk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 333c2a634..8ac3aeac0 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -222,6 +222,9 @@ class VKIE(InfoExtractor):
 
             r'<!>Видео временно недоступно':
             'Video %s is temporarily unavailable.',
+
+            r'<!>Access denied':
+            'Access denied to video %s.',
         }
 
         for error_re, error_msg in ERRORS.items():

From f72b0a603270dff0fdd72dd5218126790232199a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:15:16 +0600
Subject: [PATCH 0670/2145] Revert xvid to avi and make docs to be similar to
 existing external downloader option

---
 README.md                          | 4 ++--
 youtube_dl/YoutubeDL.py            | 3 ++-
 youtube_dl/__init__.py             | 2 +-
 youtube_dl/options.py              | 6 +++---
 youtube_dl/postprocessor/ffmpeg.py | 8 +++-----
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index d8926d2b7..9779c2058 100644
--- a/README.md
+++ b/README.md
@@ -214,8 +214,8 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --postprocessor-args             Extra parameters for video post-processor.
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)
+    --postprocessor-args ARGS        Give these arguments to the postprocessor
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8580f99a7..00af78e06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -262,7 +262,8 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    postprocessor_args: Extra parameters for external apps, like avconv.
+    postprocessor_args: A list of additional command-line arguments for the
+                        postprocessor.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index fb31d1569..2d416943f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -169,7 +169,7 @@ def _real_main(argv=None):
         if not opts.audioquality.isdigit():
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
-        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
             parser.error('invalid video recode format specified')
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c15dadb21..85365d769 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -691,11 +691,11 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
-        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)')
     postproc.add_option(
         '--postprocessor-args',
-        dest='postprocessor_args', default=None, metavar='ARGS',
-        help='Extra parameters for video post-processor.')
+        dest='postprocessor_args', metavar='ARGS',
+        help='Give these arguments to the postprocessor')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d4ba3572b..62d13a567 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -295,12 +295,10 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
-        ext = self._preferedformat
+        outpath = prefix + sep + self._preferedformat
         options = self._extra_cmd_args
-        if self._preferedformat == 'xvid':
-            ext = 'avi'
+        if self._preferedformat == 'avi':
             options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
-        outpath = prefix + sep + ext
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
@@ -308,7 +306,7 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
         self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath
         information['format'] = self._preferedformat
-        information['ext'] = ext
+        information['ext'] = self._preferedformat
         return [path], information
 
 

From e35b23f54df83886dd14428b618aa9d0221968ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:41:33 +0600
Subject: [PATCH 0671/2145] [postprocessor/common] Improve postprocessor args
 fetching and clarify doc

---
 youtube_dl/postprocessor/common.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index bee64c457..4191d040b 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -22,14 +22,15 @@ class PostProcessor(object):
     of the chain is reached.
 
     PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects. And it can receive parameters from CLI trough
-    --postprocessor-args.
+    to InfoExtractor objects.
+
+    Optionally PostProcessor can use a list of additional command-line arguments
+    with self._configuration_args.
     """
 
     _downloader = None
 
     def __init__(self, downloader=None):
-        self._extra_cmd_args = downloader.params.get('postprocessor_args')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
@@ -59,6 +60,13 @@ class PostProcessor(object):
         except Exception:
             self._downloader.report_warning(errnote)
 
+    def _configuration_args(self, default=[]):
+        pp_args = self._downloader.params.get('postprocessor_args')
+        if pp_args is None:
+            return default
+        assert isinstance(pp_args, list)
+        return pp_args
+
 
 class AudioConversionError(PostProcessingError):
     pass

From 15006fedb9511599c0e948a982fa282cde74f2ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:42:03 +0600
Subject: [PATCH 0672/2145] [postprocessor/ffmpeg] Spread postprocessor args
 usage on all ffmpeg extractors

---
 youtube_dl/postprocessor/ffmpeg.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 62d13a567..1ecce22e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -131,6 +131,8 @@ class FFmpegPostProcessor(PostProcessor):
         oldest_mtime = min(
             os.stat(encodeFilename(path)).st_mtime for path in input_paths)
 
+        opts += self._configuration_args()
+
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
@@ -294,14 +296,14 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
     def run(self, information):
         path = information['filepath']
-        prefix, sep, ext = path.rpartition('.')
-        outpath = prefix + sep + self._preferedformat
-        options = self._extra_cmd_args
-        if self._preferedformat == 'avi':
-            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
+        options = []
+        if self._preferedformat == 'avi':
+            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
+        prefix, sep, ext = path.rpartition('.')
+        outpath = prefix + sep + self._preferedformat
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath

From 369e195a44530b43bfd0e95087c9e1a238d91184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:43:02 +0600
Subject: [PATCH 0673/2145] Handle postprocessor_args similarly to
 external_downloader_args

---
 youtube_dl/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2d416943f..55b22c889 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -263,6 +263,9 @@ def _real_main(argv=None):
     external_downloader_args = None
     if opts.external_downloader_args:
         external_downloader_args = shlex.split(opts.external_downloader_args)
+    postprocessor_args = None
+    if opts.postprocessor_args:
+        postprocessor_args = shlex.split(opts.postprocessor_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))
@@ -354,7 +357,6 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
-        'postprocessor_args': shlex.split(opts.postprocessor_args or ''),
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
@@ -368,6 +370,7 @@ def _real_main(argv=None):
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'external_downloader_args': external_downloader_args,
+        'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
     }
 

From 41c0d2f8cb22fe34d957bc9b5f9032a9160685ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 23:00:19 +0600
Subject: [PATCH 0674/2145] Credit @aurium for avi recode and postprocessort
 args (#5942)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d5418dd37..c0201d875 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -130,3 +130,4 @@ Peter Ding
 jackyzy823
 George Brighton
 Remita Amine
+Aurélio A. Heckert

From ac8f97f2b37115bf81a653360819b5d30eebda18 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 11 Jul 2015 18:56:07 +0200
Subject: [PATCH 0675/2145] [rdsca] New extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rdsca.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/rdsca.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..73fa0e4ed 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,6 +460,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
+from .rdsca import RDScaIE
 from .redtube import RedTubeIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
diff --git a/youtube_dl/extractor/rdsca.py b/youtube_dl/extractor/rdsca.py
new file mode 100644
index 000000000..a8df927e4
--- /dev/null
+++ b/youtube_dl/extractor/rdsca.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    url_basename,
+)
+
+
+class RDScaIE(InfoExtractor):
+    IE_NAME = 'RDS.ca'
+    _VALID_URL = r'http://(?:www\.)?rds\.ca/videos/(?P<id>.*)'
+
+    _TESTS = [{
+        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        'info_dict': {
+            "ext": "mp4",
+            "title": "Fowler Jr. prend la direction de Jacksonville",
+            "description": "Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ",
+            "timestamp": 1430397346,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = url_basename(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'<span itemprop="name"[^>]*>([^\n]*)</span>', webpage, 'video title', default=None)
+        video_url = self._search_regex(
+            r'<span itemprop="contentURL" content="([^"]+)"', webpage, 'video URL')
+        upload_date = parse_iso8601(self._search_regex(
+            r'<span itemprop="uploadDate" content="([^"]+)"', webpage, 'upload date', default=None))
+        description = self._search_regex(
+            r'<span itemprop="description"[^>]*>([^\n]*)</span>', webpage, 'description', default=None)
+        thumbnail = self._search_regex(
+            r'<span itemprop="thumbnailUrl" content="([^"]+)"', webpage, 'upload date', default=None)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': upload_date,
+            'formats': [{
+                'url': video_url,
+            }],
+        }

From 53429e6551b80f78273f5245e11613fd39625113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 12:53:15 +0600
Subject: [PATCH 0676/2145] [onionstudios] Fix extraction

---
 youtube_dl/extractor/onionstudios.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 8fa507dec..0f1f448fe 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -49,19 +49,21 @@ class OnionStudiosIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._search_regex(
-            r'share_title\s*=\s*"([^"]+)"', webpage, 'title')
+            r'share_title\s*=\s*(["\'])(?P<title>[^\1]+?)\1',
+            webpage, 'title', group='title')
         description = self._search_regex(
-            r'share_description\s*=\s*"([^"]+)"', webpage,
-            'description', default=None)
+            r'share_description\s*=\s*(["\'])(?P<description>[^\1]+?)\1',
+            webpage, 'description', default=None, group='description')
         thumbnail = self._search_regex(
-            r'poster="([^"]+)"', webpage, 'thumbnail', default=False)
+            r'poster\s*=\s*(["\'])(?P<thumbnail>[^\1]+?)\1',
+            webpage, 'thumbnail', default=False, group='thumbnail')
 
         uploader_id = self._search_regex(
-            r'twitter_handle\s*=\s*"([^"]+)"',
-            webpage, 'uploader id', fatal=False)
+            r'twitter_handle\s*=\s*(["\'])(?P<uploader_id>[^\1]+?)\1',
+            webpage, 'uploader id', fatal=False, group='uploader_id')
         uploader = self._search_regex(
-            r'window\.channelName\s*=\s*"Embedded:([^"]+)"',
-            webpage, 'uploader', default=False)
+            r'window\.channelName\s*=\s*(["\'])Embedded:(?P<uploader>[^\1]+?)\1',
+            webpage, 'uploader', default=False, group='uploader')
 
         return {
             'id': video_id,

From 667170e2c7dee6bcf5e357dd53d461807434c5b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 20:40:00 +0600
Subject: [PATCH 0677/2145] [nrk:tv] Add support for radio URLs (Closes #6200)

---
 youtube_dl/extractor/nrk.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 9e4581cf9..eb49f292e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -116,7 +116,8 @@ class NRKPlaylistIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>https?://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    IE_DESC = 'NRK TV and NRK Radio'
+    _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -188,6 +189,10 @@ class NRKTVIE(InfoExtractor):
                 'duration': 6947.5199999999995,
             },
             'skip': 'Only works from Norway',
+        },
+        {
+            'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
+            'only_matching': True,
         }
     ]
 

From c4f1fde75bf69e05be29283c9211089e6ab269ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 20:42:18 +0600
Subject: [PATCH 0678/2145] [nrk:tv] Add format id prefixes

---
 youtube_dl/extractor/nrk.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index eb49f292e..d066a96db 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -211,7 +211,8 @@ class NRKTVIE(InfoExtractor):
         ]}
 
     def _extract_f4m(self, manifest_url, video_id):
-        return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
+        return self._extract_f4m_formats(
+            manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -273,7 +274,7 @@ class NRKTVIE(InfoExtractor):
 
         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
+            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4', m3u8_id='hls'))
         self._sort_formats(formats)
 
         subtitles_url = self._html_search_regex(

From b6ea9ef21ad3a43bddf24f5769bdf21a372381fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 23:35:56 +0600
Subject: [PATCH 0679/2145] [rds] Improve

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rds.py      | 70 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/rdsca.py    | 50 -----------------------
 3 files changed, 71 insertions(+), 51 deletions(-)
 create mode 100644 youtube_dl/extractor/rds.py
 delete mode 100644 youtube_dl/extractor/rdsca.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 73fa0e4ed..3f4f23521 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,7 +460,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
-from .rdsca import RDScaIE
+from .rds import RDSIE
 from .redtube import RedTubeIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
new file mode 100644
index 000000000..fdd3f36a6
--- /dev/null
+++ b/youtube_dl/extractor/rds.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class RDSIE(InfoExtractor):
+    IE_DESC = 'RDS.ca'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/videos/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+
+    _TEST = {
+        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        'info_dict': {
+            'id': '3.1132799',
+            'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
+            'ext': 'mp4',
+            'title': 'Fowler Jr. prend la direction de Jacksonville',
+            'description': 'Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ',
+            'timestamp': 1430397346,
+            'upload_date': '20150430',
+            'duration': 154.354,
+            'age_limit': 0,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        # TODO: extract f4m from 9c9media.com
+        video_url = self._search_regex(
+            r'<span[^>]+itemprop="contentURL"[^>]+content="([^"]+)"',
+            webpage, 'video url')
+
+        title = self._og_search_title(webpage) or self._html_search_meta(
+            'title', webpage, 'title', fatal=True)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage) or self._search_regex(
+            [r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
+             r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
+            webpage, 'thumbnail', fatal=False)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<span[^>]+itemprop="uploadDate"[^>]+content="([^"]+)"',
+            webpage, 'upload date', fatal=False))
+        duration = parse_duration(self._search_regex(
+            r'<span[^>]+itemprop="duration"[^>]+content="([^"]+)"',
+            webpage, 'duration', fatal=False))
+        age_limit = self._family_friendly_search(webpage)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'age_limit': age_limit,
+        }
diff --git a/youtube_dl/extractor/rdsca.py b/youtube_dl/extractor/rdsca.py
deleted file mode 100644
index a8df927e4..000000000
--- a/youtube_dl/extractor/rdsca.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    url_basename,
-)
-
-
-class RDScaIE(InfoExtractor):
-    IE_NAME = 'RDS.ca'
-    _VALID_URL = r'http://(?:www\.)?rds\.ca/videos/(?P<id>.*)'
-
-    _TESTS = [{
-        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
-        'info_dict': {
-            "ext": "mp4",
-            "title": "Fowler Jr. prend la direction de Jacksonville",
-            "description": "Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ",
-            "timestamp": 1430397346,
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = url_basename(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._search_regex(
-            r'<span itemprop="name"[^>]*>([^\n]*)</span>', webpage, 'video title', default=None)
-        video_url = self._search_regex(
-            r'<span itemprop="contentURL" content="([^"]+)"', webpage, 'video URL')
-        upload_date = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate" content="([^"]+)"', webpage, 'upload date', default=None))
-        description = self._search_regex(
-            r'<span itemprop="description"[^>]*>([^\n]*)</span>', webpage, 'description', default=None)
-        thumbnail = self._search_regex(
-            r'<span itemprop="thumbnailUrl" content="([^"]+)"', webpage, 'upload date', default=None)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': upload_date,
-            'formats': [{
-                'url': video_url,
-            }],
-        }

From 28fb109ed0276e85207966ac925e9885c2ed0e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 23:45:47 +0600
Subject: [PATCH 0680/2145] [rds] Improve _VALID_URL

---
 youtube_dl/extractor/rds.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
index fdd3f36a6..796adfdf9 100644
--- a/youtube_dl/extractor/rds.py
+++ b/youtube_dl/extractor/rds.py
@@ -12,9 +12,9 @@ from ..utils import (
 
 class RDSIE(InfoExtractor):
     IE_DESC = 'RDS.ca'
-    _VALID_URL = r'https?://(?:www\.)?rds\.ca/videos/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
         'info_dict': {
             'id': '3.1132799',
@@ -27,7 +27,10 @@ class RDSIE(InfoExtractor):
             'duration': 154.354,
             'age_limit': 0,
         }
-    }
+    }, {
+        'url': 'http://www.rds.ca/vid%C3%A9os/un-voyage-positif-3.877934',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 75a40b22511b4d4344cd6903e28429fa6c3991f1 Mon Sep 17 00:00:00 2001
From: Seamus Phelan <SeamusPhelan@gmail.com>
Date: Mon, 13 Jul 2015 22:35:45 +1000
Subject: [PATCH 0681/2145] [SBS] fixes due to website changes

---
 youtube_dl/extractor/sbs.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index d4bd1a0d7..ab4d1c884 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -1,12 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    remove_end,
-)
+from ..utils import remove_end
 
 
 class SBSIE(InfoExtractor):
@@ -34,18 +30,22 @@ class SBSIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        # the video is in the following iframe
+        iframe_url = 'http://www.sbs.com.au/ondemand/video/single/' + video_id + '?context=web'
+        webpage = self._download_webpage(iframe_url, video_id)
 
-        player = self._search_regex(
-            r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
-            webpage, 'player')
-        player = re.sub(r"'\s*\+\s*[\da-zA-Z_]+\s*\+\s*'", '', player)
+        player_params = self._search_regex(
+            r'(?s)(playerParams.+?releaseUrls.+?\n)',
+            webpage, 'playerParams')
+        player_params_js = self._search_regex(
+            r'({.*})',
+            player_params, 'player_param_js')
 
-        release_urls = self._parse_json(js_to_json(player), video_id)
+        player_params_json = self._parse_json(player_params_js, video_id)
 
-        theplatform_url = release_urls.get('progressive') or release_urls['standard']
+        theplatform_url = player_params_json.get('releaseUrls')['progressive'] or player_params_json.get('releaseUrls')['standard']
 
-        title = remove_end(self._og_search_title(webpage), ' (The Feed)')
+        title = remove_end(self._og_search_title(webpage, default=video_id, fatal=False), ' (The Feed)')
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 

From f870544302f75bee0d96f6a8623c8ff270beca89 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 13 Jul 2015 07:41:38 -0500
Subject: [PATCH 0682/2145] Add support for democracynow.org

Supports downloading clips or entire shows. Subtitle support
---
 youtube_dl/extractor/__init__.py     |   1 +
 youtube_dl/extractor/democracynow.py | 100 +++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/democracynow.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..5cc03b875 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -112,6 +112,7 @@ from .daum import DaumIE
 from .dbtv import DBTVIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
+from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
new file mode 100644
index 000000000..1c9b36052
--- /dev/null
+++ b/youtube_dl/extractor/democracynow.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import time
+import hmac
+import hashlib
+import itertools
+import re
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
+)
+from ..compat import compat_urllib_request
+from .common import InfoExtractor
+
+
+class DemocracynowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P<id>[^\?]*)'
+    IE_NAME = 'democracynow'
+    _TESTS = [{
+        'url': 'http://www.democracynow.org/shows/2015/7/3',
+        'info_dict': {
+            'id': '2015-0703-001',
+            'ext': 'mp4',
+            'title': 'July 03, 2015 - Democracy Now!',
+            'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
+            'uploader': 'Democracy Now',
+            'upload_date': None,
+        },
+     },{
+        'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
+        'info_dict': {
+            'id': '2015-0703-001',
+            'ext': 'mp4',
+            'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
+            'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
+            'uploader': 'Democracy Now',
+            'upload_date': None,
+        },
+
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        base_host = re.search(r'^(.+?://[^/]+)', url).group(1)
+        if display_id == '':
+            display_id = 'home'
+        webpage = self._download_webpage(url, display_id)
+        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1',webpage,re.DOTALL)
+        description = re_desc.group(2) if re_desc else ''
+
+        jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
+        js = self._parse_json(jstr, display_id)
+        video_id = None
+        formats = []
+        subtitles = {}
+        for key in ('caption_file','.......'):
+            # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
+            url = js.get(key,'')
+            if url == '' or url == None:
+                continue
+            if not re.match(r'^https?://',url):
+                url = base_host + url
+            ext = re.search(r'\.([^\.]+)$',url).group(1)
+            subtitles['eng'] = [{
+                'ext': ext,
+                'url': url,
+            }]
+        for key in ('file', 'audio'):
+            url = js.get(key,'')
+            if url == '' or url == None:
+                continue
+            if not re.match(r'^https?://',url):
+                url = base_host + url
+            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)',url)
+            if video_id == None:
+                video_id = purl.group('fn')
+            if js.get('start') != None:
+                url += '&' if purl.group('hasparams') == '?' else '?'
+                url = url + 'start='+str(js.get('start'))
+            formats.append({
+                'format_id': purl.group('dir'),
+                'ext': purl.group('ext'),
+                'url': url,
+            })
+        self._sort_formats(formats)
+        ret = {
+            'id': video_id,
+            'title': js.get('title'),
+            'description': description,
+            'uploader': 'Democracy Now',
+#            'thumbnails': thumbnails,
+            'subtitles': subtitles,
+            'formats': formats,
+        }
+        return ret
+#
\ No newline at end of file

From 6dfa0602f059397d9a41ef382d9dfa8b432ff0fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jul 2015 22:11:05 +0600
Subject: [PATCH 0683/2145] [nowtv] Fix extraction (Closes #6169)

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 173e46cd8..0b5ff4760 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -133,7 +133,7 @@ class NowTVIE(InfoExtractor):
         station = mobj.group('station')
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files' % display_id,
+            'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
             display_id)
 
         video_id = compat_str(info['id'])

From 7869eb3fc44826533d859d3e7e32df5cd4be7ef4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jul 2015 22:21:35 +0600
Subject: [PATCH 0684/2145] Credit Bernhard Minks for nowtv patch

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c0201d875..531ec5767 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -131,3 +131,4 @@ jackyzy823
 George Brighton
 Remita Amine
 Aurélio A. Heckert
+Bernhard Minks

From cfe5537ee5ca5156f3969f513a1257be83015b1d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Jul 2015 00:21:11 +0800
Subject: [PATCH 0685/2145] [myspass] Fix extraction (closes #6206)

---
 youtube_dl/extractor/myspass.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 5b9b9fbcd..4557a2b13 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -35,7 +35,8 @@ class MySpassIE(InfoExtractor):
 
         # get metadata
         metadata_url = META_DATA_URL_TEMPLATE % video_id
-        metadata = self._download_xml(metadata_url, video_id)
+        metadata = self._download_xml(
+            metadata_url, video_id, transform_source=lambda s: s.strip())
 
         # extract values from metadata
         url_flv_el = metadata.find('url_flv')

From 1d1dd597ed96790b3d041274044742baec13132e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 01:35:44 +0600
Subject: [PATCH 0686/2145] [dramafever] Extract srt (Closes #6207)

---
 youtube_dl/extractor/dramafever.py | 49 +++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index ca41a3abf..38e6597c8 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -23,8 +23,23 @@ class DramaFeverBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
 
+    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
+
+    _consumer_secret = None
+
+    def _get_consumer_secret(self):
+        mainjs = self._download_webpage(
+            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
+            None, 'Downloading main.js', fatal=False)
+        if not mainjs:
+            return self._CONSUMER_SECRET
+        return self._search_regex(
+            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
+            'consumer secret', default=self._CONSUMER_SECRET)
+
     def _real_initialize(self):
         self._login()
+        self._consumer_secret = self._get_consumer_secret()
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -119,6 +134,23 @@ class DramaFeverIE(DramaFeverBaseIE):
                 'url': href,
             }]
 
+        series_id, episode_number = video_id.split('.')
+        episode_info = self._download_json(
+            # We only need a single episode info, so restricting page size to one episode
+            # and dealing with page number as with episode number
+            r'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_number=%s&page_size=1'
+            % (self._consumer_secret, series_id, episode_number),
+            video_id, 'Downloading episode info JSON', fatal=False)
+        if episode_info:
+            value = episode_info.get('value')
+            if value:
+                subfile = value[0].get('subfile') or value[0].get('new_subfile')
+                if subfile and subfile != 'http://www.dramafever.com/st/':
+                    subtitles.setdefault('English', []).append({
+                        'ext': 'srt',
+                        'url': subfile,
+                    })
+
         return {
             'id': video_id,
             'title': title,
@@ -152,27 +184,14 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
         'playlist_count': 20,
     }]
 
-    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
     _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 
-    def _get_consumer_secret(self, video_id):
-        mainjs = self._download_webpage(
-            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
-            video_id, 'Downloading main.js', fatal=False)
-        if not mainjs:
-            return self._CONSUMER_SECRET
-        return self._search_regex(
-            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
-            'consumer secret', default=self._CONSUMER_SECRET)
-
     def _real_extract(self, url):
         series_id = self._match_id(url)
 
-        consumer_secret = self._get_consumer_secret(series_id)
-
         series = self._download_json(
             'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
-            % (consumer_secret, series_id),
+            % (self._consumer_secret, series_id),
             series_id, 'Downloading series JSON')['series'][series_id]
 
         title = clean_html(series['name'])
@@ -182,7 +201,7 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
         for page_num in itertools.count(1):
             episodes = self._download_json(
                 'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
-                % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
+                % (self._consumer_secret, series_id, self._PAGE_SIZE, page_num),
                 series_id, 'Downloading episodes JSON page #%d' % page_num)
             for episode in episodes.get('value', []):
                 episode_url = episode.get('episode_url')

From 50aa2bb6b94805d901f00d4da7a1b4ff1fcd5169 Mon Sep 17 00:00:00 2001
From: cazulu <jvlarapeinado@gmail.com>
Date: Tue, 14 Jul 2015 15:33:55 +0900
Subject: [PATCH 0687/2145] [dailymotion] Extract duration (closes #6221)

---
 youtube_dl/extractor/dailymotion.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 8852f0add..1a41c0db1 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -53,6 +53,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
                 'upload_date': '20150306',
+                'duration': 74,
             }
         },
         # Vevo video
@@ -164,6 +165,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
             'view_count': view_count,
+            'duration': info['duration']
         }
 
     def _get_subtitles(self, video_id, webpage):

From 9750e7d70eed92a6b05637465698cdd30e87a44c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 14 Jul 2015 12:56:32 +0200
Subject: [PATCH 0688/2145] [postprocessor/ffmpeg] Don't use '[youtube] ...' in
 messages

Because it can be used for other extractors.
---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1ecce22e7..1f723908b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -265,7 +265,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
         if (new_path == path or
                 (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
-            self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
+            self._downloader.to_screen('[ffmpeg] Post-process file %s exists, skipping' % new_path)
             return [], information
 
         try:

From f8da79f828637757889f3f35d7adfa9aabbfc721 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:36:30 +0600
Subject: [PATCH 0689/2145] [extractor/common] Improve _form_hidden_inputs and
 rename to _hidden_inputs

---
 youtube_dl/extractor/common.py       | 15 +++++++++++----
 youtube_dl/extractor/gorillavid.py   |  2 +-
 youtube_dl/extractor/hostingbulk.py  |  2 +-
 youtube_dl/extractor/played.py       |  2 +-
 youtube_dl/extractor/primesharetv.py |  2 +-
 youtube_dl/extractor/promptfile.py   |  2 +-
 youtube_dl/extractor/shared.py       |  2 +-
 youtube_dl/extractor/twitch.py       |  2 +-
 youtube_dl/extractor/vimeo.py        |  2 +-
 youtube_dl/extractor/vk.py           |  2 +-
 youtube_dl/extractor/vodlocker.py    |  2 +-
 11 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 82f5de2d8..315fe4a72 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -706,10 +706,17 @@ class InfoExtractor(object):
                                       'twitter card player')
 
     @staticmethod
-    def _form_hidden_inputs(html):
-        return dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            html))
+    def _hidden_inputs(html):
+        return dict([
+            (input.group('name'), input.group('value')) for input in re.finditer(
+                r'''(?x)
+                    <input\s+
+                        type=(?P<q_hidden>["\'])hidden(?P=q_hidden)\s+
+                        name=(?P<q_name>["\'])(?P<name>.+?)(?P=q_name)\s+
+                        (?:id=(?P<q_id>["\']).+?(?P=q_id)\s+)?
+                        value=(?P<q_value>["\'])(?P<value>.*?)(?P=q_value)
+                ''', html)
+        ])
 
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index aabf07a20..f006f0cb1 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -78,7 +78,7 @@ class GorillaVidIE(InfoExtractor):
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             countdown = int_or_none(self._search_regex(
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 63f579592..a3154cfde 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -58,7 +58,7 @@ class HostingBulkIE(InfoExtractor):
             r'<img src="([^"]+)".+?class="pic"',
             webpage, 'thumbnail', fatal=False)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         request = compat_urllib_request.Request(url, urlencode_postdata(fields))
         request.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 9fe1524f2..8a1c296dd 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -38,7 +38,7 @@ class PlayedIE(InfoExtractor):
         if m_error:
             raise ExtractorError(m_error.group('msg'), expected=True)
 
-        data = self._form_hidden_inputs(orig_webpage)
+        data = self._hidden_inputs(orig_webpage)
 
         self._sleep(2, video_id)
 
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 9aa0c862a..304359dc5 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -29,7 +29,7 @@ class PrimeShareTVIE(InfoExtractor):
         if '>File not exist<' in webpage:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         headers = {
             'Referer': url,
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 81a63c7fc..8190ed676 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -35,7 +35,7 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
         req = compat_urllib_request.Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 6e2b94e7d..a07677686 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -34,7 +34,7 @@ class SharedIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
-        download_form = self._form_hidden_inputs(webpage)
+        download_form = self._hidden_inputs(webpage)
         request = compat_urllib_request.Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index af2b798fb..92b6dc1b8 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -59,7 +59,7 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        login_form = self._form_hidden_inputs(login_page)
+        login_form = self._hidden_inputs(login_page)
 
         login_form.update({
             'login': username.encode('utf-8'),
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d63c03183..10d6745af 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -452,7 +452,7 @@ class VimeoChannelIE(InfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
-        fields = self._form_hidden_inputs(login_form)
+        fields = self._hidden_inputs(login_form)
         token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 8ac3aeac0..8f677cae3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -168,7 +168,7 @@ class VKIE(InfoExtractor):
         login_page = self._download_webpage(
             'https://vk.com', None, 'Downloading login page')
 
-        login_form = self._form_hidden_inputs(login_page)
+        login_form = self._hidden_inputs(login_page)
 
         login_form.update({
             'email': username.encode('cp1251'),
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 4804692bf..ccf1928b5 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -26,7 +26,7 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!

From cf61d96df0984e28d8c34328177504a2d1424bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:38:10 +0600
Subject: [PATCH 0690/2145] [extractor/common] Add _form_hidden_inputs

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 315fe4a72..3a396c0b0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -718,6 +718,12 @@ class InfoExtractor(object):
                 ''', html)
         ])
 
+    def _form_hidden_inputs(self, form_id, html):
+        form = self._search_regex(
+            r'(?s)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
+            html, '%s form' % form_id, group='form')
+        return self._hidden_inputs(form)
+
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')

From dcd4d95c8e9acec6dde0706de75bf866a43c9aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:39:41 +0600
Subject: [PATCH 0691/2145] [udemy] Fix authentication (Closes #6224)

---
 youtube_dl/extractor/udemy.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 4667ed83b..192606077 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -15,7 +15,8 @@ from ..utils import (
 class UdemyIE(InfoExtractor):
     IE_NAME = 'udemy'
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
-    _LOGIN_URL = 'https://www.udemy.com/join/login-submit/'
+    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
+    _ORIGIN_URL = 'https://www.udemy.com'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -74,29 +75,34 @@ class UdemyIE(InfoExtractor):
                 expected=True)
 
         login_popup = self._download_webpage(
-            'https://www.udemy.com/join/login-popup?displayType=ajax&showSkipButton=1', None,
-            'Downloading login popup')
+            self._LOGIN_URL, None, 'Downloading login popup')
 
         if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
             return
 
-        csrf = self._html_search_regex(
-            r'<input type="hidden" name="csrf" value="(.+?)"',
-            login_popup, 'csrf token')
+        login_form = self._form_hidden_inputs('login-form', login_popup)
 
-        login_form = {
-            'email': username,
-            'password': password,
-            'csrf': csrf,
+        login_form.update({
             'displayType': 'json',
-            'isSubmitted': '1',
-        }
+            'email': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
+        })
+
         request = compat_urllib_request.Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        response = self._download_json(
+        request.add_header('Referer', self._ORIGIN_URL)
+        request.add_header('Origin', self._ORIGIN_URL)
+
+        response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if 'returnUrl' not in response:
+        if all(logout_pattern not in response
+               for logout_pattern in ['href="https://www.udemy.com/user/logout/', '>Logout<']):
+            error = self._html_search_regex(
+                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
+                response, 'error message', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
             raise ExtractorError('Unable to log in')
 
     def _real_extract(self, url):

From 79057965a87a9179b3c3c2249df5495684c5e4ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:46:26 +0600
Subject: [PATCH 0692/2145] [udemy] Remove superfluous field

---
 youtube_dl/extractor/udemy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 192606077..e2bab52fe 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -83,7 +83,6 @@ class UdemyIE(InfoExtractor):
         login_form = self._form_hidden_inputs('login-form', login_popup)
 
         login_form.update({
-            'displayType': 'json',
             'email': username.encode('utf-8'),
             'password': password.encode('utf-8'),
         })

From 01d115b06b5b752324a9346f128d659a98b58f31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 14 Jul 2015 19:07:40 +0200
Subject: [PATCH 0693/2145] [jeuxvideo] Relax _VALID_URL (fixes #6230)

---
 youtube_dl/extractor/jeuxvideo.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index d0720ff56..1df084d87 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -8,9 +8,9 @@ from .common import InfoExtractor
 
 
 class JeuxVideoIE(InfoExtractor):
-    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)-\d+\.htm'
+    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
         'md5': '046e491afb32a8aaac1f44dd4ddd54ee',
         'info_dict': {
@@ -19,7 +19,10 @@ class JeuxVideoIE(InfoExtractor):
             'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité',
             'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.',
         },
-    }
+    }, {
+        'url': 'http://www.jeuxvideo.com/videos/chroniques/434220/l-histoire-du-jeu-video-la-saturn.htm',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From eae89f92e61d8227a6463c50750310a56c68cf09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 23:54:13 +0600
Subject: [PATCH 0694/2145] [dfb] Update test

---
 youtube_dl/extractor/dfb.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 8049779b0..6732e1751 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -10,13 +10,13 @@ class DFBIE(InfoExtractor):
     _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://tv.dfb.de/video/highlights-des-empfangs-in-berlin/9070/',
+        'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
         # The md5 is different each time
         'info_dict': {
-            'id': '9070',
+            'id': '11633',
             'ext': 'flv',
-            'title': 'Highlights des Empfangs in Berlin',
-            'upload_date': '20140716',
+            'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
+            'upload_date': '20150714',
         },
     }
 

From ddcdc684e20af96a4fc57da49c27b31c75462aae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 23:59:21 +0600
Subject: [PATCH 0695/2145] [dfb] Extract display_id and modernize

---
 youtube_dl/extractor/dfb.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 6732e1751..f1df9b9a0 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -3,17 +3,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import unified_strdate
 
 
 class DFBIE(InfoExtractor):
     IE_NAME = 'tv.dfb.de'
-    _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://tv\.dfb\.de/video/(?P<display_id>[^/]+)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
         # The md5 is different each time
         'info_dict': {
             'id': '11633',
+            'display_id': 'u-19-em-stimmen-zum-spiel-gegen-russland',
             'ext': 'flv',
             'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
             'upload_date': '20150714',
@@ -23,22 +25,25 @@ class DFBIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
         player_info = self._download_xml(
             'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id,
-            video_id)
+            display_id)
         video_info = player_info.find('video')
 
-        f4m_info = self._download_xml(self._proto_relative_url(video_info.find('url').text.strip()), video_id)
+        f4m_info = self._download_xml(
+            self._proto_relative_url(video_info.find('url').text.strip()), display_id)
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': video_info.find('title').text,
             'url': manifest_url,
             'ext': 'flv',
             'thumbnail': self._og_search_thumbnail(webpage),
-            'upload_date': ''.join(video_info.find('time_date').text.split('.')[::-1]),
+            'upload_date': unified_strdate(video_info.find('time_date').text),
         }

From 6c1b0c0ed2678198e3196e2acf32f63806d4ccb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 00:01:41 +0600
Subject: [PATCH 0696/2145] [dfb] Extract formats

---
 youtube_dl/extractor/dfb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index f1df9b9a0..263532cc6 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -37,13 +37,13 @@ class DFBIE(InfoExtractor):
             self._proto_relative_url(video_info.find('url').text.strip()), display_id)
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
+        formats = self._extract_f4m_formats(manifest_url, display_id)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': video_info.find('title').text,
-            'url': manifest_url,
-            'ext': 'flv',
             'thumbnail': self._og_search_thumbnail(webpage),
             'upload_date': unified_strdate(video_info.find('time_date').text),
+            'formats': formats,
         }

From b062d94eefcc9327af5e60f3280cedaff9e40ac8 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 03:08:36 +0600
Subject: [PATCH 0697/2145] [README.md] Clarify authentication with .netrc file

---
 README.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/README.md b/README.md
index 9779c2058..a2cc89cdb 100644
--- a/README.md
+++ b/README.md
@@ -238,6 +238,26 @@ which means you can modify it, redistribute it or use it however you like.
 
 You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
 
+### Authentication with `.netrc` file ###
+
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in shell command history. You can achieve this using [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+```
+touch $HOME/.netrc
+chmod a-rwx,u+rw $HOME/.netrc
+```
+After that you can add credentials for extractor in the following format, where *extractor* is the name of extractor in lowercase:
+```
+machine <extractor> login <login> password <password>
+```
+For example:
+```
+machine youtube login myaccount@gmail.com password my_youtube_password
+machine twitch login my_twitch_account_name password my_twitch_password
+```
+To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or to place it in [configuration file](#configuration).
+
+On Windows you may also need to setup `%HOME%` environment variable manually.
+
 # OUTPUT TEMPLATE
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:

From 2af0f87c8b56567e0254aae7a1ccbedb04413b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 23:32:52 +0600
Subject: [PATCH 0698/2145] [prosiebensat1] Fix extraction (Closes #6215)

---
 youtube_dl/extractor/prosiebensat1.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 536a42dc8..22efa903f 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,8 +9,9 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
-    unified_strdate,
+    fix_xml_ampersands,
     int_or_none,
+    unified_strdate,
 )
 
 
@@ -208,7 +209,7 @@ class ProSiebenSat1IE(InfoExtractor):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'prosieben'
-        client_name = 'kolibri-1.12.6'
+        client_name = 'kolibri-2.0.19-splec4'
         client_location = url
 
         videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
@@ -275,8 +276,9 @@ class ProSiebenSat1IE(InfoExtractor):
 
         for source in urls_sources:
             protocol = source['protocol']
+            source_url = source['url']
             if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source['url'])
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
                 if not mobj:
                     continue
                 path = mobj.group('path')
@@ -293,9 +295,18 @@ class ProSiebenSat1IE(InfoExtractor):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
+            elif 'f4mgenerator' in source_url:
+                manifest = self._download_xml(
+                    source_url, clip_id, 'Downloading generated f4m manifest',
+                    transform_source=lambda s: fix_xml_ampersands(s).strip())
+                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
+                    manifest_url = media.get('href')
+                    if manifest_url:
+                        formats.extend(self._extract_f4m_formats(
+                            manifest_url, clip_id, f4m_id='hds'))
             else:
                 formats.append({
-                    'url': source['url'],
+                    'url': source_url,
                     'vbr': fix_bitrate(source['bitrate']),
                 })
 

From 97f4aecfc1c5fd446e1d3edd37e49aafe246fe0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:14:08 +0600
Subject: [PATCH 0699/2145] [extractor/common] Handle malformed f4m manifests

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3a396c0b0..f8a5ecced 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -28,6 +28,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
+    fix_xml_ampersands,
     float_or_none,
     int_or_none,
     RegexNotFoundError,
@@ -837,7 +838,10 @@ class InfoExtractor(object):
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
-            'Unable to download f4m manifest')
+            'Unable to download f4m manifest',
+            # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+            # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
+            transform_source=lambda s: fix_xml_ampersands(s).strip())
 
         formats = []
         manifest_version = '1.0'

From cc357c4db8112ff6736a227b47fb9527d327797f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:14:52 +0600
Subject: [PATCH 0700/2145] [extractor/common] Properly handle full URLs

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f8a5ecced..78e5cf8d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -851,8 +851,10 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
-                                (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                media_url = media_el.attrib.get('href') or media_el.attrib['url']
+                manifest_url = (
+                    media_url if media_url.startswith('http://') or media_url.startswith('https://')
+                    else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From 70f0f5a8ca53d4426fc079b3ab46e9d4a8e81ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:15:15 +0600
Subject: [PATCH 0701/2145] [extractor/common] Recursively extract child f4m
 manifests

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 78e5cf8d0..e3c610aa4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,6 +27,7 @@ from ..utils import (
     bug_reports_message,
     clean_html,
     compiled_regex_type,
+    determine_ext,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
@@ -855,6 +856,13 @@ class InfoExtractor(object):
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+                # If media_url is itself a f4m manifest do the recursive extraction
+                # since bitrates in parent manifest (this one) and media_url manifest
+                # may differ leading to inability to resolve the format by requested
+                # bitrate in f4m downloader
+                if determine_ext(manifest_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From f01f731107010e0c10fc94782daa7a3ba543e92a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:15:47 +0600
Subject: [PATCH 0702/2145] [prosiebensat1] Use generic f4m manifest extraction

---
 youtube_dl/extractor/prosiebensat1.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 22efa903f..2f9d95800 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
-    fix_xml_ampersands,
+    determine_ext,
     int_or_none,
     unified_strdate,
 )
@@ -295,15 +295,8 @@ class ProSiebenSat1IE(InfoExtractor):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
-            elif 'f4mgenerator' in source_url:
-                manifest = self._download_xml(
-                    source_url, clip_id, 'Downloading generated f4m manifest',
-                    transform_source=lambda s: fix_xml_ampersands(s).strip())
-                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
-                    manifest_url = media.get('href')
-                    if manifest_url:
-                        formats.extend(self._extract_f4m_formats(
-                            manifest_url, clip_id, f4m_id='hds'))
+            elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
+                formats.extend(self._extract_f4m_formats(source_url, clip_id))
             else:
                 formats.append({
                     'url': source_url,

From 31c746e5dc46491f997eca757c5e35842f04cb59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:25:33 +0600
Subject: [PATCH 0703/2145] [extractor/common] Keep going in some media_url is
 missing

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e3c610aa4..271bf8596 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -852,7 +852,9 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                media_url = media_el.attrib.get('href') or media_el.attrib['url']
+                media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
+                if not media_url:
+                    continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))

From ab9b890b524a49a9ffa4c8ac7243cd8afc15d270 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 02:23:07 +0600
Subject: [PATCH 0704/2145] [prosiebensat1] Clarify test purpose

---
 youtube_dl/extractor/prosiebensat1.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 2f9d95800..fec008ce7 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -22,6 +22,11 @@ class ProSiebenSat1IE(InfoExtractor):
 
     _TESTS = [
         {
+            # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242
+            # in response to fixing https://github.com/rg3/youtube-dl/issues/6215:
+            # - malformed f4m manifest support
+            # - proper handling of URLs starting with `https?://` in 2.0 manifests
+            # - recursive child f4m manifests extraction
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',

From e37c932fca29d93af77b7a47cccc9bb8578e3163 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Wed, 15 Jul 2015 15:13:56 -0500
Subject: [PATCH 0705/2145] compat_urllib_parse_unquote: crash fix: only decode
 valid hex

on python 2 the following has a { "crash_rate": "100%" } of the time
as it tries to parse '" ' as hex.
---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c3783337a..1f4ccf443 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -94,6 +94,8 @@ except ImportError:
             try:
                 if not item:
                     raise ValueError
+                if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]):
+                    raise ValueError
                 pct_sequence += item[:2].decode('hex')
                 rest = item[2:]
                 if not rest:

From 45eedbe58c8ab6344f11f1e1376d01648c1967ee Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Wed, 15 Jul 2015 15:30:47 -0500
Subject: [PATCH 0706/2145] Generic: use compat_urllib_parse_unquote to prevent
 utf8 mangling of the entire page in python 2.

-requires- fixed compat_urllib_parse_unquote

example - the following will save with a mangled playlist title,
 instead of the kanji for 'tsunami'. This affects all utf8encoded
 urls as well

youtube-dl -f18 -o '%(playlist_title)s-%(title)s.%(ext)s' \
  https://gist.githubusercontent.com/atomicdryad/fcb97465e6060fc519e1/raw/61c14c1e3a4985471dcf56c281d24d7e781a4e0e/tsunami.html
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 392ad3648..fc1bf2b6e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1115,7 +1115,7 @@ class GenericIE(InfoExtractor):
         # Sometimes embedded video player is hidden behind percent encoding
         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
         # Unescaping the whole page allows to handle those cases in a generic way
-        webpage = compat_urllib_parse.unquote(webpage)
+        webpage = compat_urllib_parse_unquote(webpage)
 
         # it's tempting to parse this further, but you would
         # have to take into account all the variations like

From e118031ef827e851e537daa5b439cf5c249ca88d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:21:04 +0600
Subject: [PATCH 0707/2145] [npo] Extend _VALID_URL to support ntr.nl (Closes
 #6248)

---
 youtube_dl/extractor/npo.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 62d12b7a6..1c823ec7f 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -37,8 +37,9 @@ class NPOBaseIE(InfoExtractor):
 
 
 class NPOIE(NPOBaseIE):
-    IE_NAME = 'npo.nl'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/(?!live|radio)[^/]+/[^/]+/(?P<id>[^/?]+)'
+    IE_NAME = 'npo'
+    IE_DESC = 'npo.nl and ntr.nl'
+    _VALID_URL = r'https?://(?:www\.)?(?:npo|ntr)\.nl/(?!live|radio)(?:[^/]+/){2,}(?P<id>[^/?#]+)'
 
     _TESTS = [
         {
@@ -100,6 +101,18 @@ class NPOIE(NPOBaseIE):
                 'title': 'Hoe gaat Europa verder na Parijs?',
             },
         },
+        {
+            'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
+            'md5': '01c6a2841675995da1f0cf776f03a9c3',
+            'info_dict': {
+                'id': 'VPWON_1233944',
+                'ext': 'm4v',
+                'title': 'Aap, poot, pies',
+                'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
+                'upload_date': '20150508',
+                'duration': 599,
+            },
+        }
     ]
 
     def _real_extract(self, url):

From 525daedd5a092b0f5329952eee99a7dac5537433 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:54:43 +0600
Subject: [PATCH 0708/2145] [npo] Add support for omroepwnl fragments

---
 youtube_dl/extractor/npo.py | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 1c823ec7f..a5162c0c6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,12 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -39,7 +45,16 @@ class NPOBaseIE(InfoExtractor):
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl and ntr.nl'
-    _VALID_URL = r'https?://(?:www\.)?(?:npo|ntr)\.nl/(?!live|radio)(?:[^/]+/){2,}(?P<id>[^/?#]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                            ntr\.nl/(?:[^/]+/){2,}|
+                            omroepwnl\.nl/video/fragment/[^/]+__
+                        )
+                        (?P<id>[^/?#]+)
+                '''
 
     _TESTS = [
         {
@@ -112,6 +127,18 @@ class NPOIE(NPOBaseIE):
                 'upload_date': '20150508',
                 'duration': 599,
             },
+        },
+        {
+            'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
+            'md5': 'd30cd8417b8b9bca1fdff27428860d08',
+            'info_dict': {
+                'id': 'POW_00996502',
+                'ext': 'm4v',
+                'title': '''"Dit is wel een 'landslide'..."''',
+                'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
+                'upload_date': '20150508',
+                'duration': 462,
+            },
         }
     ]
 
@@ -127,6 +154,11 @@ class NPOIE(NPOBaseIE):
             transform_source=strip_jsonp,
         )
 
+        # For some videos actual video id (prid) is different (e.g. for
+        # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698
+        # video id is POMS_WNL_853698 but prid is POW_00996502)
+        video_id = metadata.get('prid') or video_id
+
         token = self._get_token(video_id)
 
         formats = []

From 50ea2bb20d3a3e219910e87b8b30fc79ce534595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:56:57 +0600
Subject: [PATCH 0709/2145] [npo] Update test

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index a5162c0c6..cf6a388e5 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -87,7 +87,7 @@ class NPOIE(NPOBaseIE):
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
                 'title': 'Tegenlicht',
-                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
                 'duration': 3000,
             },

From 03f32a7eadf9d832aef55673edf38023a8daff95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:14:38 +0600
Subject: [PATCH 0710/2145] [wnl] Add extractor for omroepwnl playlists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 45 +++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f4f23521..1d55275dc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -390,6 +390,7 @@ from .npo import (
     NPORadioIE,
     NPORadioFragmentIE,
     TegenlichtVproIE,
+    WNLIE
 )
 from .nrk import (
     NRKIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index cf6a388e5..c6bf7619d 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -46,12 +46,15 @@ class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl and ntr.nl'
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        (?:
-                            npo\.nl/(?!live|radio)(?:[^/]+/){2}|
-                            ntr\.nl/(?:[^/]+/){2,}|
-                            omroepwnl\.nl/video/fragment/[^/]+__
+                    (?:
+                        npo:|
+                        https?://
+                            (?:www\.)?
+                            (?:
+                                npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                                ntr\.nl/(?:[^/]+/){2,}|
+                                omroepwnl\.nl/video/fragment/[^/]+__
+                            )
                         )
                         (?P<id>[^/?#]+)
                 '''
@@ -426,3 +429,33 @@ class TegenlichtVproIE(NPOIE):
         info_page = self._download_json(
             'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
         return self._get_info(info_page['mid'])
+
+
+class WNLIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'
+
+    _TEST = {
+        'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515',
+        'info_dict': {
+            'id': 'vandaag-de-dag-6-mei',
+            'title': 'Vandaag de Dag 6 mei',
+        },
+        'playlist_count': 4,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id, 'NPO')
+            for video_id, part in re.findall(
+                r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>(Deel \d+)', webpage)
+        ]
+
+        playlist_title = self._html_search_regex(
+            r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>',
+            webpage, 'playlist title')
+
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 611ac379bb466267aded6726f9c85e79b08168c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:34:24 +0600
Subject: [PATCH 0711/2145] [vpro] Fix extraction and add support for vpro
 playlists

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/npo.py      | 35 +++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d55275dc..06f21064b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -389,7 +389,7 @@ from .npo import (
     NPOLiveIE,
     NPORadioIE,
     NPORadioFragmentIE,
-    TegenlichtVproIE,
+    VPROIE,
     WNLIE
 )
 from .nrk import (
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c6bf7619d..28d5c90b3 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -404,9 +404,8 @@ class NPORadioFragmentIE(InfoExtractor):
         }
 
 
-class TegenlichtVproIE(NPOIE):
-    IE_NAME = 'tegenlicht.vpro.nl'
-    _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
+class VPROIE(NPOIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
 
     _TESTS = [
         {
@@ -416,19 +415,35 @@ class TegenlichtVproIE(NPOIE):
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
                 'title': 'Tegenlicht',
-                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },
         },
+        {
+            'url': 'http://www.vpro.nl/programmas/2doc/2015/sergio-herman.html',
+            'info_dict': {
+                'id': 'sergio-herman',
+                'title': 'Sergio Herman: Fucking perfect',
+            },
+            'playlist_count': 2,
+        }
     ]
 
     def _real_extract(self, url):
-        name = url_basename(url)
-        webpage = self._download_webpage(url, name)
-        urn = self._html_search_meta('mediaurn', webpage)
-        info_page = self._download_json(
-            'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
-        return self._get_info(info_page['mid'])
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id, 'NPO')
+            for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
+        ]
+
+        playlist_title = self._search_regex(
+            r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>',
+            webpage, 'playlist title', default=None) or self._og_search_title(webpage)
+
+        return self.playlist_result(entries, playlist_id, playlist_title)
 
 
 class WNLIE(InfoExtractor):

From 5ba761eb854d6e415b3ab542293cb31c073dc0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:39:22 +0600
Subject: [PATCH 0712/2145] [npo] Prefer aflevering_titel over titel

---
 youtube_dl/extractor/npo.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 28d5c90b3..91adb23f0 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -234,7 +234,9 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            'title': metadata['titel'],
+            # prefer aflevering_titel if any since titel may be too generic, e.g.
+            # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
+            'title': metadata.get('aflevering_titel') or metadata['titel'],
             'description': metadata['info'],
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
@@ -414,7 +416,7 @@ class VPROIE(NPOIE):
             'info_dict': {
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
-                'title': 'Tegenlicht',
+                'title': 'De toekomst komt uit Afrika',
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },

From 536b0700b03f0b29a1025be0b7753253bd627d6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:40:04 +0600
Subject: [PATCH 0713/2145] [npo] Allow missing description

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 91adb23f0..e733d96f6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -237,7 +237,7 @@ class NPOIE(NPOBaseIE):
             # prefer aflevering_titel if any since titel may be too generic, e.g.
             # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
             'title': metadata.get('aflevering_titel') or metadata['titel'],
-            'description': metadata['info'],
+            'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),

From 574f42d79a8596ceda681b205e19e766e7bab046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:53:12 +0600
Subject: [PATCH 0714/2145] [vpro] Improve playlist extraction

---
 youtube_dl/extractor/npo.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index e733d96f6..583ed3e14 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -428,6 +428,15 @@ class VPROIE(NPOIE):
                 'title': 'Sergio Herman: Fucking perfect',
             },
             'playlist_count': 2,
+        },
+        {
+            # playlist with youtube embed
+            'url': 'http://www.vpro.nl/programmas/2doc/2015/education-education.html',
+            'info_dict': {
+                'id': 'education-education',
+                'title': '2Doc',
+            },
+            'playlist_count': 2,
         }
     ]
 
@@ -437,7 +446,7 @@ class VPROIE(NPOIE):
         webpage = self._download_webpage(url, playlist_id)
 
         entries = [
-            self.url_result('npo:%s' % video_id, 'NPO')
+            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
             for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
         ]
 

From 1540119723e7e2195a47d659993a6a3bcc02d3e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:54:05 +0600
Subject: [PATCH 0715/2145] [npo] Remove unused imports

---
 youtube_dl/extractor/npo.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 583ed3e14..f5ffe1231 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -3,17 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
     qualities,
     strip_jsonp,
     unified_strdate,
-    url_basename,
 )
 
 

From 23fc384f2c3cf9afd41fd7e033fe0823d0fc5fa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 01:28:52 +0600
Subject: [PATCH 0716/2145] [npo] Compound title

---
 youtube_dl/extractor/npo.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index f5ffe1231..0c2d02c10 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -72,7 +72,7 @@ class NPOIE(NPOBaseIE):
             'info_dict': {
                 'id': 'VARA_101191800',
                 'ext': 'm4v',
-                'title': 'De Mega Mike & Mega Thomas show',
+                'title': 'De Mega Mike & Mega Thomas show: The best of.',
                 'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
                 'upload_date': '20090227',
                 'duration': 2400,
@@ -84,7 +84,7 @@ class NPOIE(NPOBaseIE):
             'info_dict': {
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
-                'title': 'Tegenlicht',
+                'title': 'Tegenlicht: De toekomst komt uit Afrika',
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
                 'duration': 3000,
@@ -157,6 +157,13 @@ class NPOIE(NPOBaseIE):
         # video id is POMS_WNL_853698 but prid is POW_00996502)
         video_id = metadata.get('prid') or video_id
 
+        # titel is too generic in some cases so utilize aflevering_titel as well
+        # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html)
+        title = metadata['titel']
+        sub_title = metadata.get('aflevering_titel')
+        if sub_title and sub_title != title:
+            title += ': %s' % sub_title
+
         token = self._get_token(video_id)
 
         formats = []
@@ -229,9 +236,7 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            # prefer aflevering_titel if any since titel may be too generic, e.g.
-            # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
-            'title': metadata.get('aflevering_titel') or metadata['titel'],
+            'title': title,
             'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),

From a38436e8898b6eca29d9279346e6e2136ec0bc8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Jul 2015 12:02:49 +0800
Subject: [PATCH 0717/2145] [extractor/common] Add 'transform_source' parameter
 to _extract_f4m_formats()

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 271bf8596..5a2d0d995 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -836,13 +836,14 @@ class InfoExtractor(object):
         self.to_screen(msg)
         time.sleep(timeout)
 
-    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
+    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
+                             transform_source=lambda s: fix_xml_ampersands(s).strip()):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
-            transform_source=lambda s: fix_xml_ampersands(s).strip())
+            transform_source=transform_source)
 
         formats = []
         manifest_version = '1.0'

From c9c854cea7fa5992356dee5eab0d3615b4d40dc6 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:31:29 -0500
Subject: [PATCH 0718/2145] replace old compat_urllib_parse_unquote with
 backport from python3's function

  * required unquote_to_bytes function ported as well
    (uses .decode('hex') instead of dynamically populated _hextobyte global)
  * required implicit conversion to bytes and/or unicode in places due to
    differing type assumptions in p3
---
 youtube_dl/compat.py | 75 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1f4ccf443..2fd2278aa 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,10 +74,81 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+from pprint import (pprint, pformat)
+
+
+def dprint(fmt):
+    sys.stderr.write(pformat(fmt) + "\n")
+
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
-    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+    def compat_urllib_parse_unquote_to_bytes(string):
+        """unquote_to_bytes('abc%20def') -> b'abc def'."""
+        # Note: strings are encoded as UTF-8. This is only an issue if it contains
+        # unescaped non-ASCII characters, which URIs should not.
+        if not string:
+            # Is it a string-like object?
+            string.split
+            return b''
+        if isinstance(string, str):
+            string = string.encode('utf-8')
+            # string = encode('utf-8')
+
+        # python3 -> 2: must implicitly convert to bits
+        bits = bytes(string).split(b'%')
+
+        if len(bits) == 1:
+            return string
+        res = [bits[0]]
+        append = res.append
+
+        for item in bits[1:]:
+            try:
+                append(item[:2].decode('hex'))
+                append(item[2:])
+            except:
+                append(b'%')
+                append(item)
+        return b''.join(res)
+
+    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
+
+    def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+        """Replace %xx escapes by their single-character equivalent. The optional
+        encoding and errors parameters specify how to decode percent-encoded
+        sequences into Unicode characters, as accepted by the bytes.decode()
+        method.
+        By default, percent-encoded sequences are decoded with UTF-8, and invalid
+        sequences are replaced by a placeholder character.
+
+        unquote('abc%20def') -> 'abc def'.
+        """
+
+        if '%' not in string:
+            string.split
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+
+        bits = compat_urllib_parse_asciire.split(string)
+        res = [bits[0]]
+        append = res.append
+        for i in range(1, len(bits), 2):
+            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
+            foo = foo.decode(encoding, errors)
+            append(foo)
+
+            if bits[i + 1]:
+                bar = bits[i + 1]
+                if not isinstance(bar, unicode):
+                    bar = bar.decode('utf-8')
+                append(bar)
+        return ''.join(res)
+
+    def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         if string == '':
             return string
         res = string.split('%')
@@ -114,6 +185,8 @@ except ImportError:
             string += pct_sequence.decode(encoding, errors)
         return string
 
+    compat_urllib_parse_unquote = new_compat_urllib_parse_unquote
+
 try:
     compat_str = unicode  # Python 2
 except NameError:

From 851229a01f34129286a57d46f8a27b9bb5fd9a6b Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:49:55 -0500
Subject: [PATCH 0719/2145] remove debugprint

---
 youtube_dl/compat.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2fd2278aa..554e3d5db 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,12 +74,6 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
-from pprint import (pprint, pformat)
-
-
-def dprint(fmt):
-    sys.stderr.write(pformat(fmt) + "\n")
-
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:

From a0f28f90fa277d9c00f0305624dea36a20b8066e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:50:43 -0500
Subject: [PATCH 0720/2145] remove kebab

---
 youtube_dl/compat.py | 41 +----------------------------------------
 1 file changed, 1 insertion(+), 40 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 554e3d5db..de9ba2c14 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -108,7 +108,7 @@ except ImportError:
 
     compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
 
-    def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         """Replace %xx escapes by their single-character equivalent. The optional
         encoding and errors parameters specify how to decode percent-encoded
         sequences into Unicode characters, as accepted by the bytes.decode()
@@ -142,45 +142,6 @@ except ImportError:
                 append(bar)
         return ''.join(res)
 
-    def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
-        if string == '':
-            return string
-        res = string.split('%')
-        if len(res) == 1:
-            return string
-        if encoding is None:
-            encoding = 'utf-8'
-        if errors is None:
-            errors = 'replace'
-        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
-        pct_sequence = b''
-        string = res[0]
-        for item in res[1:]:
-            try:
-                if not item:
-                    raise ValueError
-                if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]):
-                    raise ValueError
-                pct_sequence += item[:2].decode('hex')
-                rest = item[2:]
-                if not rest:
-                    # This segment was just a single percent-encoded character.
-                    # May be part of a sequence of code units, so delay decoding.
-                    # (Stored in pct_sequence).
-                    continue
-            except ValueError:
-                rest = '%' + item
-            # Encountered non-percent-encoded characters. Flush the current
-            # pct_sequence.
-            string += pct_sequence.decode(encoding, errors) + rest
-            pct_sequence = b''
-        if pct_sequence:
-            # Flush the final pct_sequence
-            string += pct_sequence.decode(encoding, errors)
-        return string
-
-    compat_urllib_parse_unquote = new_compat_urllib_parse_unquote
-
 try:
     compat_str = unicode  # Python 2
 except NameError:

From 36da48798a28b8261d2f39f73f2522651d58a364 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:27:50 -0500
Subject: [PATCH 0721/2145] handle titles and captions set to ''

---
 youtube_dl/extractor/bbc.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 471d865d2..c910eb55a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -497,11 +497,13 @@ class BBCNewsIE(BBCCoUkIE):
             programme_id = jent.get('externalId')
             xml_url = jent.get('href')
 
-            title = jent.get('caption',list_title)
+            title = jent.get('caption','')
+            if title == '':
+               title = list_title
 
             duration = parse_duration(jent.get('duration'))
             description = list_title
-            if jent.get('caption'):
+            if jent.get('caption', '') != '':
                description += ' - ' + jent.get('caption')
             thumbnail = None
             if jent.has_key('image'):

From a3bfddfa5ee33cf085b959536f1025c0aa53cc77 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:47:02 -0500
Subject: [PATCH 0722/2145] bbc.py: correct syntax

---
 youtube_dl/extractor/bbc.py | 106 ++++++++++++++++++------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c910eb55a..c8f285165 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -397,14 +397,14 @@ class BBCNewsIE(BBCCoUkIE):
             'title': 'Russia stages massive WW2 parade despite Western boycott',
         },
         'playlist_count': 2,
-    },{
+    }, {
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
             'title': 'Farnborough Airshow: Video highlights',
         },
         'playlist_count': 9,
-    },{
+    }, {
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'note': 'Video',
         'info_dict': {
@@ -419,7 +419,7 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
-    },{
+    }, {
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
         'note': 'Video',
         'info_dict': {
@@ -434,7 +434,7 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
-    },{
+    }, {
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
         'note': 'Video',
         'info_dict': {
@@ -459,88 +459,88 @@ class BBCNewsIE(BBCCoUkIE):
 
         pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
         if pubdate:
-           pubdate = pubdate.replace('-','')
+            pubdate = pubdate.replace('-', '')
 
         ret = []
         jsent = []
 
         # works with bbc.com/news/something-something-123456 articles
         jsent = map(
-           lambda m: self._parse_json(m,list_id),
-           re.findall(r"data-media-meta='({[^']+})'", webpage)
+            lambda m: self._parse_json(m, list_id),
+            re.findall(r"data-media-meta='({[^']+})'", webpage)
         )
 
         if len(jsent) == 0:
-           # http://www.bbc.com/news/video_and_audio/international
-           # and single-video articles
-           masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
-           if masset:
-              jmasset = self._parse_json(masset,list_id)
-              for key, val in jmasset.get('videos',{}).items():
-                  for skey, sval in val.items():
-                      sval['id'] = skey
-                      jsent.append(sval)
+            # http://www.bbc.com/news/video_and_audio/international
+            # and single-video articles
+            masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
+            if masset:
+                jmasset = self._parse_json(masset, list_id)
+                for key, val in jmasset.get('videos', {}).items():
+                    for skey, sval in val.items():
+                        sval['id'] = skey
+                        jsent.append(sval)
 
         if len(jsent) == 0:
-           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-           # in http://www.bbc.com/news/video_and_audio/international
-           # prone to breaking if entries have sourceFiles list
-           jsent = map(
-               lambda m: self._parse_json(m,list_id),
-               re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-           )          
+            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+            # in http://www.bbc.com/news/video_and_audio/international
+            # prone to breaking if entries have sourceFiles list
+            jsent = map(
+                lambda m: self._parse_json(m, list_id),
+                re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+            )
 
         if len(jsent) == 0:
-           raise ExtractorError('No video found', expected=True)
+            raise ExtractorError('No video found', expected=True)
 
         for jent in jsent:
             programme_id = jent.get('externalId')
             xml_url = jent.get('href')
 
-            title = jent.get('caption','')
+            title = jent.get('caption', '')
             if title == '':
-               title = list_title
+                title = list_title
 
             duration = parse_duration(jent.get('duration'))
             description = list_title
             if jent.get('caption', '') != '':
-               description += ' - ' + jent.get('caption')
+                description += ' - ' + jent.get('caption')
             thumbnail = None
-            if jent.has_key('image'):
-               thumbnail=jent['image'].get('href')
+            if jent.get('image') is not None:
+                thumbnail = jent['image'].get('href')
 
             formats = []
             subtitles = []
 
             if programme_id:
-               formats, subtitles = self._download_media_selector(programme_id)
-            elif jent.has_key('sourceFiles'):
-               # mediaselector not used at
-               # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
-               for key, val in jent['sourceFiles'].items():
-                  formats.append( {
-                     'ext': val.get('encoding'),
-                     'url': val.get('url'),
-                     'filesize': int(val.get('filesize')),
-                     'format_id': key
-                  } )
+                formats, subtitles = self._download_media_selector(programme_id)
+            elif jent.get('sourceFiles') is not None:
+                # mediaselector not used at
+                # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
+                for key, val in jent['sourceFiles'].items():
+                    formats.append({
+                        'ext': val.get('encoding'),
+                        'url': val.get('url'),
+                        'filesize': int(val.get('filesize')),
+                        'format_id': key
+                    })
             elif xml_url:
-               # Cheap fallback
-               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-               formats, subtitles = self._download_media_selector(programme_id)
+                # Cheap fallback
+                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+                formats, subtitles = self._download_media_selector(programme_id)
 
             if len(formats) == 0:
-               raise ExtractorError('unsupported json media entry.\n    '+str(jent)+'\n')
-               
+                raise ExtractorError('unsupported json media entry.\n    ' + str(jent) + '\n')
+
             self._sort_formats(formats)
 
-            id = jent.get('id') if programme_id == None else programme_id
-            if id == None:
-               id = 'NA'
+            id = jent.get('id') if programme_id is None else programme_id
+            if id is None:
+                id = 'NA'
 
-            ret.append( {
+            ret.append({
                 'id': id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
@@ -550,8 +550,8 @@ class BBCNewsIE(BBCCoUkIE):
                 'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
-            } )
+            })
 
         if len(ret) > 0:
-           return self.playlist_result(ret, list_id, list_title)
+            return self.playlist_result(ret, list_id, list_title)
         raise ExtractorError('No video found', expected=True)

From eb08081330f5ef52d66140589137ae1bb05eee5f Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:57:08 -0500
Subject: [PATCH 0723/2145] democracynow: correct syntax

---
 youtube_dl/extractor/democracynow.py | 43 +++++++++-------------------
 1 file changed, 14 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 1c9b36052..973bb437b 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -1,19 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import time
-import hmac
-import hashlib
-import itertools
 import re
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_age_limit,
-    parse_iso8601,
-)
-from ..compat import compat_urllib_request
 from .common import InfoExtractor
 
 
@@ -30,7 +18,7 @@ class DemocracynowIE(InfoExtractor):
             'uploader': 'Democracy Now',
             'upload_date': None,
         },
-     },{
+    }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
         'info_dict': {
             'id': '2015-0703-001',
@@ -40,7 +28,6 @@ class DemocracynowIE(InfoExtractor):
             'uploader': 'Democracy Now',
             'upload_date': None,
         },
-
     }]
 
     def _real_extract(self, url):
@@ -49,7 +36,7 @@ class DemocracynowIE(InfoExtractor):
         if display_id == '':
             display_id = 'home'
         webpage = self._download_webpage(url, display_id)
-        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1',webpage,re.DOTALL)
+        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1', webpage, re.DOTALL)
         description = re_desc.group(2) if re_desc else ''
 
         jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
@@ -57,30 +44,30 @@ class DemocracynowIE(InfoExtractor):
         video_id = None
         formats = []
         subtitles = {}
-        for key in ('caption_file','.......'):
+        for key in ('caption_file', '.......'):
             # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
-            url = js.get(key,'')
-            if url == '' or url == None:
+            url = js.get(key, '')
+            if url == '' or url is None:
                 continue
-            if not re.match(r'^https?://',url):
+            if not re.match(r'^https?://', url):
                 url = base_host + url
-            ext = re.search(r'\.([^\.]+)$',url).group(1)
+            ext = re.search(r'\.([^\.]+)$', url).group(1)
             subtitles['eng'] = [{
                 'ext': ext,
                 'url': url,
             }]
         for key in ('file', 'audio'):
-            url = js.get(key,'')
-            if url == '' or url == None:
+            url = js.get(key, '')
+            if url == '' or url is None:
                 continue
-            if not re.match(r'^https?://',url):
+            if not re.match(r'^https?://', url):
                 url = base_host + url
-            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)',url)
-            if video_id == None:
+            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)', url)
+            if video_id is None:
                 video_id = purl.group('fn')
-            if js.get('start') != None:
+            if js.get('start') is not None:
                 url += '&' if purl.group('hasparams') == '?' else '?'
-                url = url + 'start='+str(js.get('start'))
+                url = url + 'start=' + str(js.get('start'))
             formats.append({
                 'format_id': purl.group('dir'),
                 'ext': purl.group('ext'),
@@ -92,9 +79,7 @@ class DemocracynowIE(InfoExtractor):
             'title': js.get('title'),
             'description': description,
             'uploader': 'Democracy Now',
-#            'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
         }
         return ret
-#
\ No newline at end of file

From 9fefc88656eceac13604fd86dfb25dc736ed239a Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 07:24:07 -0500
Subject: [PATCH 0724/2145] fix TestCompat test_all_present

---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index de9ba2c14..8b4d0287c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -451,7 +451,9 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
+    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urlparse',

From 593b77064c51c411071e310578b542017b9b2ec8 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 09:45:49 -0500
Subject: [PATCH 0725/2145] Don't forget trailing '%'

---
 youtube_dl/compat.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8b4d0287c..9e506352f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -98,6 +98,9 @@ except ImportError:
         append = res.append
 
         for item in bits[1:]:
+            if item == '':
+                append(b'%')
+                continue
             try:
                 append(item[:2].decode('hex'))
                 append(item[2:])

From 4a632911443f0dbc2384fb82ade85382aeecc8dc Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 09:46:08 -0500
Subject: [PATCH 0726/2145] Add tests for compat_urllib_parse_unquote

---
 test/test_compat.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 1eb454e06..431e6bdf1 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
+    compat_urllib_parse_unquote,
 )
 
 
@@ -42,5 +43,24 @@ class TestCompat(unittest.TestCase):
             dir(youtube_dl.compat))) - set(['unicode_literals'])
         self.assertEqual(all_names, sorted(present_names))
 
+    def test_compat_urllib_parse_unquote(self):
+        test_strings = [
+            ['''''', ''''''],
+            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
+            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
+            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
+             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
+            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
+             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
+        ]
+        for test in test_strings:
+            strutf = test[0]
+            strurlenc = test[1]
+            strurldec = compat_urllib_parse_unquote(strurlenc)
+            self.assertEqual(strutf, strurldec)
+            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+
 if __name__ == '__main__':
     unittest.main()

From cabe001590914338524dff0c1edffbde4a1447ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:18:33 +0100
Subject: [PATCH 0727/2145] [howstuffwoks] fix _VALID_URL regex

---
 youtube_dl/extractor/howstuffworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index e97339121..f59393150 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(\d+-)*(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
             'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',

From f354385bf50e04b0e7e1defcc14264d1a66e0a07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:43:27 +0100
Subject: [PATCH 0728/2145] Add test for urls without a number

---
 youtube_dl/extractor/howstuffworks.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index f59393150..c9b6579a6 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -46,6 +46,17 @@ class HowStuffWorksIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        {
+            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
+            'info_dict': {
+                'id': '855410',
+                'ext': 'mp4',
+                'title': 'Stuff to Blow Your Mind',
+                'description': 'When it comes to optical illusions it’s pretty easy to jailbreak the operating system of your mind and load a new program onto it. Find out why your visual system willingly makes errors in interpretation.',
+                'display_id': 'optical-illusions',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        }
     ]
 
     def _real_extract(self, url):

From 1186e3f91ab940017be263c65c27b821992e58cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:45:53 +0100
Subject: [PATCH 0729/2145] do not capture a group

---
 youtube_dl/extractor/howstuffworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index c9b6579a6..cc931d571 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(\d+-)*(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
             'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',

From c4fe07c7afcc59dfcd5cfa4626ad37deb19e1033 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 17:07:55 +0100
Subject: [PATCH 0730/2145] match only the test url

---
 youtube_dl/extractor/howstuffworks.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index cc931d571..663e6632a 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -48,14 +48,7 @@ class HowStuffWorksIE(InfoExtractor):
         },
         {
             'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
-            'info_dict': {
-                'id': '855410',
-                'ext': 'mp4',
-                'title': 'Stuff to Blow Your Mind',
-                'description': 'When it comes to optical illusions it’s pretty easy to jailbreak the operating system of your mind and load a new program onto it. Find out why your visual system willingly makes errors in interpretation.',
-                'display_id': 'optical-illusions',
-                'thumbnail': 're:^https?://.*\.jpg$',
-            },
+            'only_matching': True,
         }
     ]
 

From 55139679261f8c2409ca150906a2693731452a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 22:58:13 +0600
Subject: [PATCH 0731/2145] [compat] Simplify and use latest cpython 3 code

---
 youtube_dl/compat.py | 41 +++++++++++++----------------------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e506352f..54ccf1d28 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -75,8 +75,13 @@ except ImportError:
     import BaseHTTPServer as compat_http_server
 
 try:
+    from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
-except ImportError:
+except ImportError:  # Python 2
+    # HACK: The following are the correct unquote_to_bytes and unquote
+    # implementations from cpython 3.4.3's stdlib. Python 2's version
+    # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
+
     def compat_urllib_parse_unquote_to_bytes(string):
         """unquote_to_bytes('abc%20def') -> b'abc def'."""
         # Note: strings are encoded as UTF-8. This is only an issue if it contains
@@ -85,32 +90,22 @@ except ImportError:
             # Is it a string-like object?
             string.split
             return b''
-        if isinstance(string, str):
+        if isinstance(string, unicode):
             string = string.encode('utf-8')
-            # string = encode('utf-8')
-
-        # python3 -> 2: must implicitly convert to bits
-        bits = bytes(string).split(b'%')
-
+        bits = string.split(b'%')
         if len(bits) == 1:
             return string
         res = [bits[0]]
         append = res.append
-
         for item in bits[1:]:
-            if item == '':
-                append(b'%')
-                continue
             try:
-                append(item[:2].decode('hex'))
+                append(compat_urllib_parse._hextochr[item[:2]])
                 append(item[2:])
-            except:
+            except KeyError:
                 append(b'%')
                 append(item)
         return b''.join(res)
 
-    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
-
     def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         """Replace %xx escapes by their single-character equivalent. The optional
         encoding and errors parameters specify how to decode percent-encoded
@@ -121,7 +116,6 @@ except ImportError:
 
         unquote('abc%20def') -> 'abc def'.
         """
-
         if '%' not in string:
             string.split
             return string
@@ -129,20 +123,12 @@ except ImportError:
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-
-        bits = compat_urllib_parse_asciire.split(string)
+        bits = compat_urllib_parse._asciire.split(string)
         res = [bits[0]]
         append = res.append
         for i in range(1, len(bits), 2):
-            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
-            foo = foo.decode(encoding, errors)
-            append(foo)
-
-            if bits[i + 1]:
-                bar = bits[i + 1]
-                if not isinstance(bar, unicode):
-                    bar = bar.decode('utf-8')
-                append(bar)
+            append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
+            append(bits[i + 1])
         return ''.join(res)
 
 try:
@@ -454,7 +440,6 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
-    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',

From 14309e1ddc476a7e2fc444a0443b2fc23186a385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 22:58:39 +0600
Subject: [PATCH 0732/2145] [test_compat] Make tests more idiomatic

---
 test/test_compat.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 431e6bdf1..2ffbc2c48 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -44,23 +44,22 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):
-        test_strings = [
-            ['''''', ''''''],
-            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
-            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
-            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
-%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
-             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
-%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
-            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
-             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
-        ]
-        for test in test_strings:
-            strutf = test[0]
-            strurlenc = test[1]
-            strurldec = compat_urllib_parse_unquote(strurlenc)
-            self.assertEqual(strutf, strurldec)
-            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+        self.assertEqual(compat_urllib_parse_unquote(''), '')
+        self.assertEqual(compat_urllib_parse_unquote('%'), '%')
+        self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
+        self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%')
+        self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
+        self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),
+            '''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
+            '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
 if __name__ == '__main__':
     unittest.main()

From d79febcd0619608aa100c17b0a2aefe9f4836d9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:09:56 +0600
Subject: [PATCH 0733/2145] [test_compat] Remove redundant test

---
 test/test_compat.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 2ffbc2c48..d816a9236 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -51,7 +51,6 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
         self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
         self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
-        self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波')
         self.assertEqual(
             compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
 %<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),

From aa99aa4e853966d4f01e280313e3aadf2ddf7ad6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:28:34 +0600
Subject: [PATCH 0734/2145] [compat] Add compat_urllib_parse_unquote_plus

---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 54ccf1d28..a3a2aef53 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -77,6 +77,7 @@ except ImportError:
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
+    from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
     # HACK: The following are the correct unquote_to_bytes and unquote
     # implementations from cpython 3.4.3's stdlib. Python 2's version
@@ -131,6 +132,15 @@ except ImportError:  # Python 2
             append(bits[i + 1])
         return ''.join(res)
 
+    def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
+        """Like unquote(), but also replace plus signs by spaces, as required for
+        unquoting HTML form values.
+
+        unquote_plus('%7e/abc+def') -> '~/abc def'
+        """
+        string = string.replace('+', ' ')
+        return compat_urllib_parse_unquote(string, encoding, errors)
+
 try:
     compat_str = unicode  # Python 2
 except NameError:
@@ -441,6 +451,7 @@ __all__ = [
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_plus',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',

From 8954e481404746c65ae1e82a6c5edec69114bfb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:31:23 +0600
Subject: [PATCH 0735/2145] [test_compat] Add tests for
 compat_urllib_parse_unquote_plus

---
 test/test_compat.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index d816a9236..c3ba8ad2e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -15,6 +15,7 @@ from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 
 
@@ -44,6 +45,8 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):
+        self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
         self.assertEqual(compat_urllib_parse_unquote(''), '')
         self.assertEqual(compat_urllib_parse_unquote('%'), '%')
         self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
@@ -60,5 +63,9 @@ class TestCompat(unittest.TestCase):
             compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
             '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
+    def test_compat_urllib_parse_unquote_plus(self):
+        self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
+
 if __name__ == '__main__':
     unittest.main()

From 4d08161ac23cfdf563b5d79b9f75cde6e6e57178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:32:43 +0600
Subject: [PATCH 0736/2145] [compat] Mention unquote_plus

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3a2aef53..e950a4688 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,7 +79,7 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
-    # HACK: The following are the correct unquote_to_bytes and unquote
+    # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
 

From b94b78971ce70a999584acab57d0a620a91bc2aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:35:14 +0600
Subject: [PATCH 0737/2145] [bet] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/bet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 26b934543..03dad4636 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     xpath_text,
     xpath_with_ns,
@@ -57,7 +57,7 @@ class BetIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        media_url = compat_urllib_parse.unquote(self._search_regex(
+        media_url = compat_urllib_parse_unquote(self._search_regex(
             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
             webpage, 'media URL'))
 

From 3e72f5f10e8dbac71e267ce10e791251a54e61ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:36:11 +0600
Subject: [PATCH 0738/2145] [ceskatelevize] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/ceskatelevize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 65f6be623..dda583680 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -88,7 +89,7 @@ class CeskaTelevizeIE(InfoExtractor):
         if playlist_url == 'error_region':
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlist_url))
+        req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
         playlist = self._download_json(req, video_id)

From a60cccbf9f76e7e42509bf1ed4e054643ecc348f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:36:58 +0600
Subject: [PATCH 0739/2145] [crunchyroll] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/crunchyroll.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 73f1e22ef..d1b6d7366 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -12,6 +12,7 @@ from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -254,7 +255,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             video_upload_date = unified_strdate(video_upload_date)
         video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
 
-        playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
+        playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = compat_urllib_request.Request(playerdata_url)
         playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')

From d3671b344f663c3aa82aea86de51a70f24511792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:37:23 +0600
Subject: [PATCH 0740/2145] [ehow] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/ehow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index 9cb1bf301..ba9685991 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from .common import InfoExtractor
 
@@ -26,7 +26,7 @@ class EHowIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
             r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
-        final_url = compat_urllib_parse.unquote(video_url)
+        final_url = compat_urllib_parse_unquote(video_url)
         uploader = self._html_search_meta('uploader', webpage)
         title = self._og_search_title(webpage).replace(' | eHow', '')
 

From d7011316d0bce88796a6d33e895fdafa231fd8ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:37:56 +0600
Subject: [PATCH 0741/2145] [facebook] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/facebook.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 82dc27bc6..e17bb9aea 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_http_client,
     compat_str,
     compat_urllib_error,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -136,7 +136,7 @@ class FacebookIE(InfoExtractor):
             else:
                 raise ExtractorError('Cannot parse data')
         data = dict(json.loads(m.group(1)))
-        params_raw = compat_urllib_parse.unquote(data['params'])
+        params_raw = compat_urllib_parse_unquote(data['params'])
         params = json.loads(params_raw)
         video_data = params['video_data'][0]
 

From 1f80e360fce9fc057847379ec06d0e635f697198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:38:30 +0600
Subject: [PATCH 0742/2145] [gamespot] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/gamespot.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 2d33fa7f5..b3f1bafcc 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -5,7 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -75,7 +75,7 @@ class GameSpotIE(InfoExtractor):
         return {
             'id': data_video['guid'],
             'display_id': page_id,
-            'title': compat_urllib_parse.unquote(data_video['title']),
+            'title': compat_urllib_parse_unquote(data_video['title']),
             'formats': formats,
             'description': self._html_search_meta('description', webpage),
             'thumbnail': self._og_search_thumbnail(webpage),

From f7e6f7fa2374864ba28fbe6840a0462b7c1033db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:39:32 +0600
Subject: [PATCH 0743/2145] [extractor/generic] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/generic.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fc1bf2b6e..a62287e50 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -8,7 +8,6 @@ import re
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
     compat_urlparse,
@@ -1369,7 +1368,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
         if mobj is not None:
-            return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
+            return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
 
         # Look for funnyordie embed
         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
@@ -1682,7 +1681,7 @@ class GenericIE(InfoExtractor):
         entries = []
         for video_url in found:
             video_url = compat_urlparse.urljoin(url, video_url)
-            video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
+            video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
 
             # Sometimes, jwplayer extraction will result in a YouTube URL
             if YoutubeIE.suitable(video_url):

From 8ee4ecb48dd7824f451c11c4cebb05b603a2da27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:39:54 +0600
Subject: [PATCH 0744/2145] [infoq] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/infoq.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 91a1b3ccb..71cfd12c5 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -4,7 +4,7 @@ import base64
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 
@@ -39,7 +39,7 @@ class InfoQIE(InfoExtractor):
         # Extract video URL
         encoded_id = self._search_regex(
             r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id')
-        real_id = compat_urllib_parse.unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
+        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
         playpath = 'mp4:' + real_id
 
         video_filename = playpath.split('/')[-1]

From 899a3e2f139a7b774d1a9483bb975e480e642c75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:40:35 +0600
Subject: [PATCH 0745/2145] [karaoketv] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/karaoketv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index e3b43ff8d..06daf5a89 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
     js_to_json,
 )
@@ -24,7 +24,7 @@ class KaraoketvIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         page_video_url = self._og_search_video_url(webpage, video_id)
-        config_json = compat_urllib_parse.unquote_plus(self._search_regex(
+        config_json = compat_urllib_parse_unquote_plus(self._search_regex(
             r'config=(.*)', page_video_url, 'configuration'))
 
         urls_info_json = self._download_json(

From 977a247a0639728146dd5c0d369b542e0a1af69b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:41:14 +0600
Subject: [PATCH 0746/2145] [malemotion] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/malemotion.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
index 0b85a59d1..92511a671 100644
--- a/youtube_dl/extractor/malemotion.py
+++ b/youtube_dl/extractor/malemotion.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class MalemotionIE(InfoExtractor):
@@ -24,7 +22,7 @@ class MalemotionIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_url = compat_urllib_parse.unquote(self._search_regex(
+        video_url = compat_urllib_parse_unquote(self._search_regex(
             r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)</title', webpage, 'title')

From c177bb3a50cde6035976603ba8094dd72cc310fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:41:47 +0600
Subject: [PATCH 0747/2145] [metacafe] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/metacafe.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 8bc333b02..6e2e73a51 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -155,7 +156,7 @@ class MetacafeIE(InfoExtractor):
         video_url = None
         mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
         if mobj is not None:
-            mediaURL = compat_urllib_parse.unquote(mobj.group(1))
+            mediaURL = compat_urllib_parse_unquote(mobj.group(1))
             video_ext = mediaURL[-3:]
 
             # Extract gdaKey if available

From 09b718c4398b2ce9964ec3ae5e329278d6211668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:42:11 +0600
Subject: [PATCH 0748/2145] [mitele] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mitele.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 7091f3335..852d72266 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -5,6 +5,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -48,7 +49,7 @@ class MiTeleIE(InfoExtractor):
             domain = 'http://' + domain
         info_url = compat_urlparse.urljoin(
             domain,
-            compat_urllib_parse.unquote(embed_data['flashvars']['host'])
+            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
         )
         info_el = self._download_xml(info_url, episode).find('./video/info')
 

From c2daf8dfa44b07179b272874c336d2435a2ff653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:42:43 +0600
Subject: [PATCH 0749/2145] [mixcloud] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mixcloud.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 425a4ccf1..d47aeceda 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -3,9 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
@@ -60,7 +58,7 @@ class MixcloudIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group(1)
         cloudcast_name = mobj.group(2)
-        track_id = compat_urllib_parse.unquote('-'.join((uploader, cloudcast_name)))
+        track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name)))
 
         webpage = self._download_webpage(url, track_id)
 

From e97bb3de838219a8c2800f3eb646313977503af2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:43:36 +0600
Subject: [PATCH 0750/2145] [mofosex] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mofosex.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index 2cec12d35..9bf99a54a 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -5,9 +5,9 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
 )
 
 
@@ -34,7 +34,7 @@ class MofosexIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, 'title')
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
+        video_url = compat_urllib_parse_unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[5].split('_')[:2]

From 9fd3bf04b7b160c1f24f9818689075f2570b8a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:44:38 +0600
Subject: [PATCH 0751/2145] [myvideo] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/myvideo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 5e754fcff..c96f472a3 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_ord,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -107,7 +108,7 @@ class MyVideoIE(InfoExtractor):
             if not a == '_encxml':
                 params[a] = b
             else:
-                encxml = compat_urllib_parse.unquote(b)
+                encxml = compat_urllib_parse_unquote(b)
         if not params.get('domain'):
             params['domain'] = 'www.myvideo.de'
         xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
@@ -135,7 +136,7 @@ class MyVideoIE(InfoExtractor):
         video_url = None
         mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
         if mobj:
-            video_url = compat_urllib_parse.unquote(mobj.group(1))
+            video_url = compat_urllib_parse_unquote(mobj.group(1))
             if 'myvideo2flash' in video_url:
                 self.report_warning(
                     'Rewriting URL to use unencrypted rtmp:// ...',
@@ -147,10 +148,10 @@ class MyVideoIE(InfoExtractor):
             mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
             if mobj is None:
                 raise ExtractorError('unable to extract url')
-            video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2))
+            video_url = compat_urllib_parse_unquote(mobj.group(1)) + compat_urllib_parse_unquote(mobj.group(2))
 
         video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
-        video_file = compat_urllib_parse.unquote(video_file)
+        video_file = compat_urllib_parse_unquote(video_file)
 
         if not video_file.endswith('f4m'):
             ppath, prefix = video_file.split('.')
@@ -159,7 +160,7 @@ class MyVideoIE(InfoExtractor):
             video_playpath = ''
 
         video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
-        video_swfobj = compat_urllib_parse.unquote(video_swfobj)
+        video_swfobj = compat_urllib_parse_unquote(video_swfobj)
 
         video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
                                               webpage, 'title')

From b78f5ec4c338e5e9379c1d7a2b0c99451e06897a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:00 +0600
Subject: [PATCH 0752/2145] [odnoklassniki] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/odnoklassniki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 6c7149fe3..215ffe87b 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     unified_strdate,
     int_or_none,
@@ -62,7 +62,7 @@ class OdnoklassnikiIE(InfoExtractor):
             metadata = self._parse_json(metadata, video_id)
         else:
             metadata = self._download_json(
-                compat_urllib_parse.unquote(flashvars['metadataUrl']),
+                compat_urllib_parse_unquote(flashvars['metadataUrl']),
                 video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']

From d54f1c74776a8d6c7cf07ae16ae67bd39ee51f93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:26 +0600
Subject: [PATCH 0753/2145] [openfilm] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/openfilm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
index 2249657eb..7e10bdf2c 100644
--- a/youtube_dl/extractor/openfilm.py
+++ b/youtube_dl/extractor/openfilm.py
@@ -5,7 +5,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote_plus,
     parse_age_limit,
     int_or_none,
 )
@@ -37,7 +37,7 @@ class OpenFilmIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        player = compat_urllib_parse.unquote_plus(
+        player = compat_urllib_parse_unquote_plus(
             self._og_search_video_url(webpage))
 
         video = json.loads(self._search_regex(

From 2ebbb6f1f771e5c81b103b6216073341e85cc1aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:55 +0600
Subject: [PATCH 0754/2145] [photobucket] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/photobucket.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index c66db3cdc..788411ccc 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -4,7 +4,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 
 
 class PhotobucketIE(InfoExtractor):
@@ -34,7 +34,7 @@ class PhotobucketIE(InfoExtractor):
         info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
                                        webpage, 'info json')
         info = json.loads(info_json)
-        url = compat_urllib_parse.unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
+        url = compat_urllib_parse_unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
         return {
             'id': video_id,
             'url': url,

From 388ad0c05c976a769ae9cf9d1f0e59526921b68c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:46:33 +0600
Subject: [PATCH 0755/2145] [playvid] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/playvid.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index c3e667e9e..2eb4fd96d 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -4,7 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     clean_html,
@@ -44,7 +45,7 @@ class PlayvidIE(InfoExtractor):
         flashvars = self._html_search_regex(
             r'flashvars="(.+?)"', webpage, 'flashvars')
 
-        infos = compat_urllib_parse.unquote(flashvars).split(r'&')
+        infos = compat_urllib_parse_unquote(flashvars).split(r'&')
         for info in infos:
             videovars_match = re.match(r'^video_vars\[(.+?)\]=(.+?)$', info)
             if videovars_match:
@@ -52,7 +53,7 @@ class PlayvidIE(InfoExtractor):
                 val = videovars_match.group(2)
 
                 if key == 'title':
-                    video_title = compat_urllib_parse.unquote_plus(val)
+                    video_title = compat_urllib_parse_unquote_plus(val)
                 if key == 'duration':
                     try:
                         duration = int(val)

From 605cbef65326b52ba9fdb994fc06798f15a3a1ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:47:23 +0600
Subject: [PATCH 0756/2145] [pornhub] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/pornhub.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 8172bc997..0b7886840 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -5,7 +5,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
@@ -69,7 +70,7 @@ class PornHubIE(InfoExtractor):
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:
-            thumbnail = compat_urllib_parse.unquote(thumbnail)
+            thumbnail = compat_urllib_parse_unquote(thumbnail)
 
         view_count = self._extract_count(
             r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
@@ -80,9 +81,9 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse.unquote_plus(
+            password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 

From 47af21e8f15be1ddb6195327719569d8723a669e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:47:51 +0600
Subject: [PATCH 0757/2145] [spankwire] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/spankwire.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index bff75d6b2..5fa6faf18 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
@@ -68,7 +68,7 @@ class SpankwireIE(InfoExtractor):
             webpage, 'comment count', fatal=False))
 
         video_urls = list(map(
-            compat_urllib_parse.unquote,
+            compat_urllib_parse_unquote,
             re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(

From 736f003f2e96b37a9554d77a26fdef6c0a38fd5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:49:14 +0600
Subject: [PATCH 0758/2145] [xbef] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xbef.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 80c48c37d..4ff99e5ca 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class XBefIE(InfoExtractor):
@@ -30,7 +28,7 @@ class XBefIE(InfoExtractor):
         config_url_enc = self._download_webpage(
             'http://xbef.com/Main/GetVideoURLEncoded/%s' % video_id, video_id,
             note='Retrieving config URL')
-        config_url = compat_urllib_parse.unquote(config_url_enc)
+        config_url = compat_urllib_parse_unquote(config_url_enc)
         config = self._download_xml(
             config_url, video_id, note='Retrieving config')
 

From 7dde5f6a8d10ba98b43f920c73248a8a7e73dc38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:49:38 +0600
Subject: [PATCH 0759/2145] [xnxx] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xnxx.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 79ed6c744..5a41f8ffa 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class XNXXIE(InfoExtractor):
@@ -26,7 +24,7 @@ class XNXXIE(InfoExtractor):
 
         video_url = self._search_regex(r'flv_url=(.*?)&amp;',
                                        webpage, 'video URL')
-        video_url = compat_urllib_parse.unquote(video_url)
+        video_url = compat_urllib_parse_unquote(video_url)
 
         video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM',
                                               webpage, 'title')

From ee8de13e14f0dee555c340ef8b9f9c15a6bac6dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:02 +0600
Subject: [PATCH 0760/2145] [xtube] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 1644f53c8..779e4f46a 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -5,7 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from ..utils import (
     parse_duration,
@@ -59,7 +59,7 @@ class XTubeIE(InfoExtractor):
         for format_id, video_url in re.findall(
                 r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
             fmt = {
-                'url': compat_urllib_parse.unquote(video_url),
+                'url': compat_urllib_parse_unquote(video_url),
                 'format_id': format_id,
             }
             m = re.search(r'^(?P<height>\d+)[pP]', format_id)
@@ -68,7 +68,7 @@ class XTubeIE(InfoExtractor):
             formats.append(fmt)
 
         if not formats:
-            video_url = compat_urllib_parse.unquote(self._search_regex(
+            video_url = compat_urllib_parse_unquote(self._search_regex(
                 r'flashvars\.video_url\s*=\s*"([^"]+)"',
                 webpage, 'video URL'))
             formats.append({'url': video_url})

From aa4789d632b71abb79561ecf6769258a66e158f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:25 +0600
Subject: [PATCH 0761/2145] [xvideos] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xvideos.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index d8415bed4..5dcf2fdd1 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -37,7 +37,7 @@ class XVideosIE(InfoExtractor):
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        video_url = compat_urllib_parse.unquote(
+        video_url = compat_urllib_parse_unquote(
             self._search_regex(r'flv_url=(.+?)&', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')

From db6c50f109e732f287cc0c61216cdd865a031ea3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:52 +0600
Subject: [PATCH 0762/2145] [ynet] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/ynet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 894678a23..869f3e819 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -5,7 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote_plus
 
 
 class YnetIE(InfoExtractor):
@@ -34,7 +34,7 @@ class YnetIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage))
+        content = compat_urllib_parse_unquote_plus(self._og_search_video_url(webpage))
         config = json.loads(self._search_regex(r'config=({.+?})$', content, 'video config'))
         f4m_url = config['clip']['url']
         title = self._og_search_title(webpage)

From 7fd002c0061f9f96eca2224553a2551748970e7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:51:57 +0600
Subject: [PATCH 0763/2145] [youtube] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/youtube.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3c629d38a..e7f5c7861 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -17,6 +17,8 @@ from ..compat import (
     compat_chr,
     compat_parse_qs,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
@@ -865,7 +867,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
         if mobj:
-            url = proto + '://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
+            url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
         video_id = self.extract_id(url)
 
         # Get video webpage
@@ -973,7 +975,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # uploader
         if 'author' not in video_info:
             raise ExtractorError('Unable to extract uploader name')
-        video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
+        video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
 
         # uploader_id
         video_uploader_id = None
@@ -1000,7 +1002,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning('unable to extract video thumbnail')
             video_thumbnail = None
         else:   # don't panic if we can't find it
-            video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
+            video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
 
         # upload date
         upload_date = self._html_search_meta(
@@ -1062,7 +1064,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning('unable to extract video duration')
             video_duration = None
         else:
-            video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
+            video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))
 
         # annotations
         video_annotations = None
@@ -1609,7 +1611,7 @@ class YoutubeSearchURLIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        query = compat_urllib_parse.unquote_plus(mobj.group('query'))
+        query = compat_urllib_parse_unquote_plus(mobj.group('query'))
 
         webpage = self._download_webpage(url, query)
         result_code = self._search_regex(

From c60e8cfaf7405e8b8075f06a62e7547adad8beba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:54:38 +0600
Subject: [PATCH 0764/2145] [ehow] Simplify

---
 youtube_dl/extractor/ehow.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index ba9685991..b1cd4f5d4 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-from ..compat import (
-    compat_urllib_parse_unquote,
-)
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 
 
 class EHowIE(InfoExtractor):

From 7bd42d0d963b543aae1f4b4e8c157dc3421189fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:56:27 +0600
Subject: [PATCH 0765/2145] [openfilm] Fix compat_urllib_parse_unquote_plus
 import

---
 youtube_dl/extractor/openfilm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
index 7e10bdf2c..d2ceedd01 100644
--- a/youtube_dl/extractor/openfilm.py
+++ b/youtube_dl/extractor/openfilm.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
     parse_iso8601,
-    compat_urllib_parse_unquote_plus,
     parse_age_limit,
     int_or_none,
 )

From 6b19647d5706b75b5e2d7da836495fe5174919f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 00:04:25 +0600
Subject: [PATCH 0766/2145] [veehd] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/veehd.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 346edf485..0d8d832cc 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -5,6 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -76,7 +77,7 @@ class VeeHDIE(InfoExtractor):
 
         if config_json:
             config = json.loads(config_json)
-            video_url = compat_urlparse.unquote(config['clip']['url'])
+            video_url = compat_urllib_parse_unquote(config['clip']['url'])
 
         if not video_url:
             video_url = self._html_search_regex(

From 3cc8b4c327585ff4dbb045d17d6f5c160df6fdb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 00:24:39 +0600
Subject: [PATCH 0767/2145] [compat] Fix missing _asciire on python 2.6

---
 youtube_dl/compat.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e950a4688..db0da5828 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,6 +79,8 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
+    _asciire = re.compile('([\x00-\x7f]+)') if sys.version_info < (2, 7) else compat_urllib_parse._asciire
+
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
@@ -124,7 +126,7 @@ except ImportError:  # Python 2
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-        bits = compat_urllib_parse._asciire.split(string)
+        bits = _asciire.split(string)
         res = [bits[0]]
         append = res.append
         for i in range(1, len(bits), 2):

From 9441f77faaf078aca2947b7613f61f46b6d0aea2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 01:45:36 +0600
Subject: [PATCH 0768/2145] [bliptv] Use xpath for extraction and fix test

---
 youtube_dl/extractor/bliptv.py | 51 +++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index fb56cd78d..cd06dbcea 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -14,6 +14,8 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     unescapeHTML,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
@@ -23,10 +25,10 @@ class BlipTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352',
-            'md5': 'c6934ad0b6acf2bd920720ec888eb812',
+            'md5': '80baf1ec5c3d2019037c1c707d676b9f',
             'info_dict': {
                 'id': '5779306',
-                'ext': 'mov',
+                'ext': 'm4v',
                 'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3',
                 'description': 'md5:9bc31f227219cde65e47eeec8d2dc596',
                 'timestamp': 1323138843,
@@ -128,35 +130,34 @@ class BlipTVIE(InfoExtractor):
 
         rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS')
 
-        def blip(s):
-            return '{http://blip.tv/dtd/blip/1.0}%s' % s
-
-        def media(s):
-            return '{http://search.yahoo.com/mrss/}%s' % s
-
-        def itunes(s):
-            return '{http://www.itunes.com/dtds/podcast-1.0.dtd}%s' % s
+        def _x(p):
+            return xpath_with_ns(p, {
+                'blip': 'http://blip.tv/dtd/blip/1.0',
+                'media': 'http://search.yahoo.com/mrss/',
+                'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
+            })
 
         item = rss.find('channel/item')
 
-        video_id = item.find(blip('item_id')).text
-        title = item.find('./title').text
-        description = clean_html(compat_str(item.find(blip('puredescription')).text))
-        timestamp = parse_iso8601(item.find(blip('datestamp')).text)
-        uploader = item.find(blip('user')).text
-        uploader_id = item.find(blip('userid')).text
-        duration = int(item.find(blip('runtime')).text)
-        media_thumbnail = item.find(media('thumbnail'))
-        thumbnail = media_thumbnail.get('url') if media_thumbnail is not None else item.find(itunes('image')).text
-        categories = [category.text for category in item.findall('category')]
+        video_id = xpath_text(item, _x('blip:item_id'), 'video id') or lookup_id
+        title = xpath_text(item, 'title', 'title', fatal=True)
+        description = clean_html(xpath_text(item, _x('blip:puredescription'), 'description'))
+        timestamp = parse_iso8601(xpath_text(item, _x('blip:datestamp'), 'timestamp'))
+        uploader = xpath_text(item, _x('blip:user'), 'uploader')
+        uploader_id = xpath_text(item, _x('blip:userid'), 'uploader id')
+        duration = int_or_none(xpath_text(item, _x('blip:runtime'), 'duration'))
+        media_thumbnail = item.find(_x('media:thumbnail'))
+        thumbnail = (media_thumbnail.get('url') if media_thumbnail is not None
+                     else xpath_text(item, 'image', 'thumbnail'))
+        categories = [category.text for category in item.findall('category') if category is not None]
 
         formats = []
         subtitles_urls = {}
 
-        media_group = item.find(media('group'))
-        for media_content in media_group.findall(media('content')):
+        media_group = item.find(_x('media:group'))
+        for media_content in media_group.findall(_x('media:content')):
             url = media_content.get('url')
-            role = media_content.get(blip('role'))
+            role = media_content.get(_x('blip:role'))
             msg = self._download_webpage(
                 url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url',
                 video_id, 'Resolving URL for %s' % role)
@@ -175,8 +176,8 @@ class BlipTVIE(InfoExtractor):
                     'url': real_url,
                     'format_id': role,
                     'format_note': media_type,
-                    'vcodec': media_content.get(blip('vcodec')) or 'none',
-                    'acodec': media_content.get(blip('acodec')),
+                    'vcodec': media_content.get(_x('blip:vcodec')) or 'none',
+                    'acodec': media_content.get(_x('blip:acodec')),
                     'filesize': media_content.get('filesize'),
                     'width': int_or_none(media_content.get('width')),
                     'height': int_or_none(media_content.get('height')),

From 530857182df72db028f959c0a58a9daf219d6b22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 01:45:54 +0600
Subject: [PATCH 0769/2145] [bliptv] Add test with missing duration

---
 youtube_dl/extractor/bliptv.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index cd06dbcea..a69ee482b 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -102,6 +102,20 @@ class BlipTVIE(InfoExtractor):
                 'vcodec': 'none',
             }
         },
+        {
+            # missing duration
+            'url': 'http://blip.tv/rss/flash/6700880',
+            'info_dict': {
+                'id': '6684191',
+                'ext': 'm4v',
+                'title': 'Cowboy Bebop: Gateway Shuffle Review',
+                'description': 'md5:3acc480c0f9ae157f5fe88547ecaf3f8',
+                'timestamp': 1386639757,
+                'upload_date': '20131210',
+                'uploader': 'sfdebris',
+                'uploader_id': '706520',
+            }
+        }
     ]
 
     @staticmethod

From 3c283a381e4f7a69bf57c3ea85aab3c85ce0e309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 02:43:18 +0600
Subject: [PATCH 0770/2145] [sbs] Simplify

---
 youtube_dl/extractor/sbs.py | 40 +++++++++++++++----------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index ab4d1c884..d6ee2d9e2 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -2,12 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import remove_end
 
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/(?:single/)?(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P<id>[0-9]+)'
 
     _TESTS = [{
         # Original URL is handled by the generic IE which finds the iframe:
@@ -17,43 +16,36 @@ class SBSIE(InfoExtractor):
         'info_dict': {
             'id': '320403011771',
             'ext': 'mp4',
-            'title': 'Dingo Conservation',
-            'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction',
+            'title': 'Dingo Conservation (The Feed)',
+            'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
             'thumbnail': 're:http://.*\.jpg',
+            'duration': 308,
         },
-        'add_ies': ['generic'],
     }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
         'only_matching': True,
+    }, {
+        'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # the video is in the following iframe
-        iframe_url = 'http://www.sbs.com.au/ondemand/video/single/' + video_id + '?context=web'
-        webpage = self._download_webpage(iframe_url, video_id)
+        webpage = self._download_webpage(
+            'http://www.sbs.com.au/ondemand/video/single/%s?context=web' % video_id, video_id)
 
-        player_params = self._search_regex(
-            r'(?s)(playerParams.+?releaseUrls.+?\n)',
-            webpage, 'playerParams')
-        player_params_js = self._search_regex(
-            r'({.*})',
-            player_params, 'player_param_js')
+        player_params = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+playerParams\s*=\s*({.+?});', webpage, 'playerParams'),
+            video_id)
 
-        player_params_json = self._parse_json(player_params_js, video_id)
-
-        theplatform_url = player_params_json.get('releaseUrls')['progressive'] or player_params_json.get('releaseUrls')['standard']
-
-        title = remove_end(self._og_search_title(webpage, default=video_id, fatal=False), ' (The Feed)')
-        description = self._html_search_meta('description', webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        urls = player_params['releaseUrls']
+        theplatform_url = (urls.get('progressive') or urls.get('standard') or
+                           urls.get('html') or player_params['relatedItemsURL'])
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
             'url': theplatform_url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
         }

From 4e1ad6e9a86e03a4309ed63914a08484aaff2784 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:13:45 +0600
Subject: [PATCH 0771/2145] [videomega] Extend _VALID_URL and improve
 extraction (Closes #6260)

---
 youtube_dl/extractor/videomega.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index eb309a7cd..d45f88646 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,17 +8,14 @@ from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://
-        (?:www\.)?videomega\.tv/
-        (?:iframe\.php|cdn\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
-        '''
+    _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
     _TEST = {
-        'url': 'http://videomega.tv/?ref=4GNA688SU99US886ANG4',
-        'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
-            'id': '4GNA688SU99US886ANG4',
+            'id': 'AOSQBJYKIDDIKYJBQSOA',
             'ext': 'mp4',
-            'title': 'BigBuckBunny_320x180',
+            'title': '1254207',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -26,15 +23,15 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
-        req = compat_urllib_request.Request(iframe_url)
+        req = compat_urllib_request.Request(url)
         req.add_header('Referer', url)
+        req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
 
         title = self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title')
+            r'<title>(.+?)</title>', webpage, 'title')
         title = re.sub(
-            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s?|\s?-\svideomega\.tv$)', '', title)
+            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
             r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
         video_url = self._search_regex(
@@ -46,6 +43,6 @@ class VideoMegaIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
-                'Referer': iframe_url,
+                'Referer': url,
             },
         }

From 8a37f53685c3a9227aad139c9faf13df8576abb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:25:10 +0600
Subject: [PATCH 0772/2145] [videomega] Revert iframe URL

---
 youtube_dl/extractor/videomega.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index d45f88646..42d42adb0 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -10,7 +10,7 @@ from ..compat import compat_urllib_request
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
     _TEST = {
-        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
             'id': 'AOSQBJYKIDDIKYJBQSOA',
@@ -23,7 +23,8 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
+        req = compat_urllib_request.Request(iframe_url)
         req.add_header('Referer', url)
         req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
@@ -43,6 +44,6 @@ class VideoMegaIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
-                'Referer': url,
+                'Referer': iframe_url,
             },
         }

From 4bdfef5a1874d99457cf5758c5d8e1cc0c487f08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:25:30 +0600
Subject: [PATCH 0773/2145] [videomega] Add tests

---
 youtube_dl/extractor/videomega.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 42d42adb0..b2ef637fa 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -9,7 +9,7 @@ from ..compat import compat_urllib_request
 
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
@@ -18,7 +18,13 @@ class VideoMegaIE(InfoExtractor):
             'title': '1254207',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomega.tv/view.php?ref=090051111052065112106089103052052103089106112065052111051090',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 4211e1941b39cb153505c64ab8fcb31a762674d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:27:09 +0600
Subject: [PATCH 0774/2145] [videomega] Add shortcut to _VALID_URL

---
 youtube_dl/extractor/videomega.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index b2ef637fa..78ff6310a 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',

From 1793d71db61acc9f6cceefbc63d7de2b558b7582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:18:03 +0600
Subject: [PATCH 0775/2145] [twitch:stream] Fix channel_id in different case
 (Closes #6263)

---
 youtube_dl/extractor/twitch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 92b6dc1b8..ab557a1a5 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -346,6 +346,8 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
+        channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
+
         access_token = self._download_json(
             '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
             'Downloading channel access token')

From 06966677343c4bbb51e8da3f0ed4d4985bdf54f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:27:45 +0600
Subject: [PATCH 0776/2145] [twitch:stream] Clarify channel_id reassignment
 rationale (#6263)

---
 youtube_dl/extractor/twitch.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index ab557a1a5..948c8ce39 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -346,6 +346,10 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
+        # Channel name may be typed if different case than the original channel name
+        # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
+        # an invalid m3u8 URL. Working around by use of original channel name from stream
+        # JSON and fallback to lowercase if it's not available.
         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
 
         access_token = self._download_json(

From f29ac588ddb5b36ebfe38e3a809a70cf0cb543a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:33:50 +0600
Subject: [PATCH 0777/2145] [nationalgeographics] Fix extraction (Closes #6262)

---
 youtube_dl/extractor/nationalgeographic.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index c18640c5a..f793b72f5 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -25,8 +25,11 @@ class NationalGeographicIE(InfoExtractor):
         name = url_basename(url)
 
         webpage = self._download_webpage(url, name)
-        feed_url = self._search_regex(r'data-feed-url="([^"]+)"', webpage, 'feed url')
-        guid = self._search_regex(r'data-video-guid="([^"]+)"', webpage, 'guid')
+        feed_url = self._search_regex(
+            r'data-feed-url="([^"]+)"', webpage, 'feed url')
+        guid = self._search_regex(
+            r'id="(?:videoPlayer|player-container)"[^>]+data-guid="([^"]+)"',
+            webpage, 'guid')
 
         feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
         content = feed.find('.//{http://search.yahoo.com/mrss/}content')

From 1b541d8d6ee5e3543253501d725af7ce4a55cad8 Mon Sep 17 00:00:00 2001
From: sceext <sceext@foxmail.com>
Date: Sat, 18 Jul 2015 12:29:35 +0800
Subject: [PATCH 0778/2145] [iqiyi] fix iqiyi (2015-07-17), update the md5 salt
 (enc_key) to iqiyi latest (2015-07-17) flash player.

---
 youtube_dl/extractor/iqiyi.py | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 0f6707d7c..ac5416953 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -212,20 +212,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        filename, _ = os.path.splitext(url_basename(swf_url))
-        enc_key_json = self._downloader.cache.load('iqiyi-enc-key', filename)
-        if enc_key_json is not None:
-            return enc_key_json[0]
-
-        req = self._request_webpage(
-            swf_url, video_id, note='download swf content')
-        cn = req.read()
-        cn = zlib.decompress(cn[8:])
-        pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
-        enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
-
-        self._downloader.cache.store('iqiyi-enc-key', filename, [enc_key])
-
+        enc_key = '8e29ab5666d041c3a1ea76e06dabdffb'
         return enc_key
 
     def _real_extract(self, url):

From b0bff54b08a44966d321e8de0ed08dc1dde6b96e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 18 Jul 2015 11:59:45 +0200
Subject: [PATCH 0779/2145] release 2015.07.18

---
 docs/supportedsites.md | 58 +++++++++++++++++++++++++++---------------
 youtube_dl/version.py  |  2 +-
 2 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0ca06c71d..a84878026 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,7 +28,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
- - **AppleDaily**
+ - **AppleDaily**: 臺灣蘋果日報
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -45,7 +45,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
- - **BaiduVideo**
+ - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
  - **Bandcamp**
@@ -106,7 +106,7 @@
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
- - **CtsNews**
+ - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
  - **dailymotion**
  - **dailymotion:playlist**
@@ -121,7 +121,7 @@
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
- - **DouyuTV**
+ - **DouyuTV**: 斗鱼
  - **dramafever**
  - **dramafever:series**
  - **DRBonanza**
@@ -222,7 +222,7 @@
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
  - **IPrima**
- - **iqiyi**
+ - **iqiyi**: 爱奇艺
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -243,9 +243,15 @@
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
+ - **kuwo:album**: 酷我音乐 - 专辑
+ - **kuwo:category**: 酷我音乐 - 分类
+ - **kuwo:chart**: 酷我音乐 - 排行榜
+ - **kuwo:mv**: 酷我音乐 - MV
+ - **kuwo:singer**: 酷我音乐 - 歌手
+ - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
- - **Letv**
+ - **Letv**: 乐视网
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
@@ -297,6 +303,7 @@
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
+ - **Myvi**
  - **myvideo**
  - **MyVidster**
  - **N-JOY**
@@ -312,11 +319,18 @@
  - **NDTV**
  - **NerdCubedFeed**
  - **Nerdist**
+ - **netease:album**: 网易云音乐 - 专辑
+ - **netease:djradio**: 网易云音乐 - 电台
+ - **netease:mv**: 网易云音乐 - MV
+ - **netease:playlist**: 网易云音乐 - 歌单
+ - **netease:program**: 网易云音乐 - 电台节目
+ - **netease:singer**: 网易云音乐 - 歌手
+ - **netease:song**: 网易云音乐
  - **Netzkino**
  - **Newgrounds**
  - **Newstube**
- - **NextMedia**
- - **NextMediaActionNews**
+ - **NextMedia**: 蘋果日報
+ - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
@@ -332,13 +346,14 @@
  - **Nowness**
  - **NowTV**
  - **nowvideo**: NowVideo
- - **npo.nl**
+ - **npo**: npo.nl and ntr.nl
+ - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
  - **NRK**
  - **NRKPlaylist**
- - **NRKTV**
+ - **NRKTV**: NRK TV and NRK Radio
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
@@ -382,11 +397,11 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
- - **qqmusic**
- - **qqmusic:album**
- - **qqmusic:playlist**
- - **qqmusic:singer**
- - **qqmusic:toplist**
+ - **qqmusic**: QQ音乐
+ - **qqmusic:album**: QQ音乐 - 专辑
+ - **qqmusic:playlist**: QQ音乐 - 歌单
+ - **qqmusic:singer**: QQ音乐 - 歌手
+ - **qqmusic:toplist**: QQ音乐 - 排行榜
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -395,6 +410,7 @@
  - **RadioJavan**
  - **Rai**
  - **RBMARadio**
+ - **RDS**: RDS.ca
  - **RedTube**
  - **Restudy**
  - **ReverbNation**
@@ -495,7 +511,6 @@
  - **TechTalks**
  - **techtv.mit.edu**
  - **ted**
- - **tegenlicht.vpro.nl**
  - **TeleBruxelles**
  - **telecinco.es**
  - **TeleMB**
@@ -551,7 +566,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
- - **UDNEmbed**
+ - **UDNEmbed**: 聯合影音
  - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
@@ -613,9 +628,11 @@
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
  - **WebOfStories**
+ - **WebOfStoriesPlaylist**
  - **Weibo**
  - **Wimp**
  - **Wistia**
+ - **WNL**
  - **WorldStarHipHop**
  - **wrzuta.pl**
  - **WSJ**: Wall Street Journal
@@ -628,18 +645,19 @@
  - **Xstream**
  - **XTube**
  - **XTubeUser**: XTube user profile
- - **Xuite**
+ - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
- - **Yam**
+ - **Yam**: 蕃薯藤yam天空部落
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
  - **YesJapan**
+ - **yinyuetai:video**: 音悦Tai
  - **Ynet**
  - **YouJizz**
- - **youku**
+ - **youku**: 优酷
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3364647ed..3ad7a2bc0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.07'
+__version__ = '2015.07.18'

From 74fe23ec35483c552f9c253be4c565546f78f001 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 16:35:28 +0600
Subject: [PATCH 0780/2145] [extractor/common] Style

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2d0d995..b9014fc23 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -996,7 +996,7 @@ class InfoExtractor(object):
     def _parse_smil_video(self, video, video_id, base, rtmp_count):
         src = video.get('src')
         if not src:
-            return ([], rtmp_count)
+            return [], rtmp_count
         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
         width = int_or_none(video.get('width'))
         height = int_or_none(video.get('height'))
@@ -1009,7 +1009,7 @@ class InfoExtractor(object):
                     proto = 'http'
         ext = video.get('ext')
         if proto == 'm3u8':
-            return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
+            return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
         elif proto == 'rtmp':
             rtmp_count += 1
             streamer = video.get('streamer') or base

From dc786d3db59a611cbcb716a476602719e74a34e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:22:25 +0600
Subject: [PATCH 0781/2145] [vk:uservideos] Improve extraction

---
 youtube_dl/extractor/vk.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 8f677cae3..93c5b18c9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -291,23 +291,32 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk.com:user-videos'
     IE_DESC = 'vk.com:All of a user\'s videos'
-    _VALID_URL = r'https?://vk\.com/videos(?P<id>[0-9]+)(?:m\?.*)?'
+    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
     _TEMPLATE_URL = 'https://vk.com/videos'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://vk.com/videos205387401',
         'info_dict': {
             'id': '205387401',
+            'title': "Tom Cruise's Videos",
         },
         'playlist_mincount': 4,
-    }
+    }, {
+        'url': 'http://vk.com/videos-77521',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
-        page = self._download_webpage(url, page_id)
-        video_ids = orderedSet(
-            m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
-        url_entries = [
+
+        webpage = self._download_webpage(url, page_id)
+
+        entries = [
             self.url_result(
                 'http://vk.com/video' + video_id, 'VK', video_id=video_id)
-            for video_id in video_ids]
-        return self.playlist_result(url_entries, page_id)
+            for video_id in set(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
+
+        title = unescapeHTML(self._search_regex(
+            r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',
+            webpage, 'title', default=page_id))
+
+        return self.playlist_result(entries, page_id, title)

From 1ecb5d1d83970d6295c655a37c99e390fb5d7dee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:23:33 +0600
Subject: [PATCH 0782/2145] [vk] Clarify extractor names

---
 youtube_dl/extractor/vk.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 93c5b18c9..ef94b6894 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -20,7 +20,8 @@ from ..utils import (
 
 
 class VKIE(InfoExtractor):
-    IE_NAME = 'vk.com'
+    IE_NAME = 'vk'
+    IE_DESC = 'VK'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
@@ -289,8 +290,8 @@ class VKIE(InfoExtractor):
 
 
 class VKUserVideosIE(InfoExtractor):
-    IE_NAME = 'vk.com:user-videos'
-    IE_DESC = 'vk.com:All of a user\'s videos'
+    IE_NAME = 'vk:uservideos'
+    IE_DESC = "VK - User's Videos"
     _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{

From c6b68648f45498bcacc71cc8f29696fa93259a7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:36:46 +0600
Subject: [PATCH 0783/2145] [bilibili] Show georestriction error

---
 youtube_dl/extractor/bilibili.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index bf60450c2..ecc17ebeb 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -41,8 +41,15 @@ class BiliBiliIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        if self._search_regex(r'(此视频不存在或被删除)', webpage, 'error message', default=None):
-            raise ExtractorError('The video does not exist or was deleted', expected=True)
+        if '(此视频不存在或被删除)' in webpage:
+            raise ExtractorError(
+                'The video does not exist or was deleted', expected=True)
+
+        if '>你没有权限浏览！ 由于版权相关问题 我们不对您所在的地区提供服务<' in webpage:
+            raise ExtractorError(
+                'The video is not available in your region due to copyright reasons',
+                expected=True)
+
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 

From fec73daaa30d006bc6c5d0483d255b7bc1c256b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 18:05:08 +0600
Subject: [PATCH 0784/2145] [vk:uservideos] Revert orderedSet

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ef94b6894..98bd3a141 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -314,7 +314,7 @@ class VKUserVideosIE(InfoExtractor):
         entries = [
             self.url_result(
                 'http://vk.com/video' + video_id, 'VK', video_id=video_id)
-            for video_id in set(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
+            for video_id in orderedSet(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
 
         title = unescapeHTML(self._search_regex(
             r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',

From 22603348aa0b3e02c520589dea092507a04ab06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 18:32:52 +0600
Subject: [PATCH 0785/2145] [compat] Fix _asciire

---
 youtube_dl/compat.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index db0da5828..0c57c7aeb 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,7 +79,8 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
-    _asciire = re.compile('([\x00-\x7f]+)') if sys.version_info < (2, 7) else compat_urllib_parse._asciire
+    _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
+                else re.compile('([\x00-\x7f]+)'))
 
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version

From 9ac09ed4deafc77cfd94e6a8827e4b0c47517b5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:13:10 +0600
Subject: [PATCH 0786/2145] [bliptv] Remove unused import

---
 youtube_dl/extractor/bliptv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index a69ee482b..c3296283d 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 
 from ..compat import (
-    compat_str,
     compat_urllib_request,
     compat_urlparse,
 )

From ee48b6a88f4b8f076699f539ee4c92389fa37643 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:15:20 +0600
Subject: [PATCH 0787/2145] [vk] Capture error message

---
 youtube_dl/extractor/vk.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 98bd3a141..9487fc9f5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -205,6 +205,12 @@ class VKIE(InfoExtractor):
 
         info_page = self._download_webpage(info_url, video_id)
 
+        error_message = self._html_search_regex(
+            r'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>',
+            info_page, 'error message', default=None)
+        if error_message:
+            raise ExtractorError(error_message, expected=True)
+
         if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
             raise ExtractorError(
                 'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',

From e58066e244bf9a04c2853e8829f7939bdefc9084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:25:06 +0600
Subject: [PATCH 0788/2145] [vk] Add age restricted video test for reference

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9487fc9f5..c30c5a8e5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -154,6 +154,11 @@ class VKIE(InfoExtractor):
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'only_matching': True,
         },
+        {
+            # age restricted video, requires vk account credentials
+            'url': 'https://vk.com/video205387401_164765225',
+            'only_matching': True,
+        },
         {
             # vk wrapper
             'url': 'http://www.biqle.ru/watch/847655_160197695',

From bfed4813b224f720e716de4d4f27471ca9053a0b Mon Sep 17 00:00:00 2001
From: linhua55 <5linhua5@gmail.com>
Date: Sat, 18 Jul 2015 23:33:42 +0800
Subject: [PATCH 0789/2145] fix extraction

 for http://www.tudou.com/albumplay/cJAHGih4yYg.html
---
 youtube_dl/extractor/tudou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c89de5ba4..8095e18d2 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/?.*/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',

From 3f5c6d0c1b955f9312b83fb0e1f3ae59d1230c82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:41:34 +0600
Subject: [PATCH 0790/2145] [francetv] Add support for embeds and clean up
 _VALID_URL

---
 youtube_dl/extractor/francetv.py | 44 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index b2c984bf2..827400172 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -160,11 +160,19 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
 class FranceTVIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetv'
     IE_DESC = 'France 2, 3, 4, 5 and Ô'
-    _VALID_URL = r'''(?x)https?://www\.france[2345o]\.fr/
-        (?:
-            emissions/.*?/(videos|emissions)/(?P<id>[^/?]+)
-        |   (emissions?|jt)/(?P<key>[^/?]+)
-        )'''
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?france[2345o]\.fr/
+                                (?:
+                                    emissions/[^/]+/(?:videos|diffusions)?|
+                                    videos
+                                )
+                            /|
+                            embed\.francetv\.fr/\?ue=
+                        )
+                        (?P<id>[^/?]+)
+                    '''
 
     _TESTS = [
         # france2
@@ -232,13 +240,33 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'timestamp': 1410822000,
             },
         },
+        {
+            # francetv embed
+            'url': 'http://embed.francetv.fr/?ue=8d7d3da1e3047c42ade5a5d7dfd3fc87',
+            'info_dict': {
+                'id': 'EV_30231',
+                'ext': 'flv',
+                'title': 'Alcaline, le concert avec Calogero',
+                'description': 'md5:',
+                'upload_date': '20150226',
+                'timestamp': 1424989860,
+            },
+        },
+        {
+            'url': 'http://www.france4.fr/emission/highlander/diffusion-du-17-07-2015-04h05',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.franceo.fr/videos/125377617',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        webpage = self._download_webpage(url, mobj.group('key') or mobj.group('id'))
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
         video_id, catalogue = self._html_search_regex(
-            r'href="http://videos\.francetv\.fr/video/([^@]+@[^"]+)"',
+            r'href="http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
             webpage, 'video ID').split('@')
         return self._extract_video(video_id, catalogue)
 

From 5705ee6ef8d49175bf4afac36af4b3b9a4e72596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:43:42 +0600
Subject: [PATCH 0791/2145] [francetv] Fix duration extraction

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 827400172..d4f98ca16 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -87,7 +87,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'title': info['titre'],
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
-            'duration': float_or_none(info.get('real_duration'), 1000) or parse_duration(info['duree']),
+            'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
         }
@@ -250,6 +250,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'description': 'md5:',
                 'upload_date': '20150226',
                 'timestamp': 1424989860,
+                'duration': 5400,
             },
         },
         {

From cbd55ade680c5a3ffe5f7b5d36dc01af2c6dc48d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:56:00 +0600
Subject: [PATCH 0792/2145] [extractor/generic] Add support for francetv embeds

---
 youtube_dl/extractor/generic.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a62287e50..dc24a8a8b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -407,6 +407,26 @@ class GenericIE(InfoExtractor):
                 'skip_download': 'Requires rtmpdump'
             }
         },
+        # francetv embed
+        {
+            'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
+            'info_dict': {
+                'id': 'EV_30231',
+                'ext': 'mp4',
+                'title': 'Alcaline, le concert avec Calogero',
+                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
+                'upload_date': '20150226',
+                'timestamp': 1424989860,
+                'duration': 5400,
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            },
+            'expected_warnings': [
+                'Forbidden'
+            ]
+        },
         # Condé Nast embed
         {
             'url': 'http://www.wired.com/2014/04/honda-asimo/',
@@ -1431,6 +1451,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
 
+        # Look for embedded francetv player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for embedded smotri.com player
         smotri_url = SmotriIE._extract_url(webpage)
         if smotri_url:

From 826a7da808448932d17c09a13a106d75ba0c10d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 00:27:23 +0600
Subject: [PATCH 0793/2145] [rtlnl] Avoid episodes completely (Closes #6275)

---
 youtube_dl/extractor/rtlnl.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index a4d3d73ff..e0c530d64 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -43,6 +43,21 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
+        'info_dict': {
+            'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',
+            'ext': 'mp4',
+            'title': 'RTL Nieuws - Meer beelden van overval juwelier',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
+            'timestamp': 1437233400,
+            'upload_date': '20150718',
+            'duration': 30.474,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # encrypted m3u8 streams, georestricted
         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
@@ -59,9 +74,11 @@ class RtlNlIE(InfoExtractor):
             uuid)
 
         material = info['material'][0]
-        progname = info['abstracts'][0]['name']
-        subtitle = material['title'] or info['episodes'][0]['name']
-        description = material.get('synopsis') or info['episodes'][0]['synopsis']
+        title = info['abstracts'][0]['name']
+        subtitle = material.get('title')
+        if subtitle:
+            title += ' - %s' % subtitle
+        description = material.get('synopsis')
 
         meta = info.get('meta', {})
 
@@ -107,7 +124,7 @@ class RtlNlIE(InfoExtractor):
 
         return {
             'id': uuid,
-            'title': '%s - %s' % (progname, subtitle),
+            'title': title,
             'formats': formats,
             'timestamp': material['original_date'],
             'description': description,

From 761ee0d827d17eefff0538a99c453e050b6e933e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 02:28:43 +0800
Subject: [PATCH 0794/2145] [iqiyi] Remove unused imports

---
 youtube_dl/extractor/iqiyi.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ac5416953..7caaa1183 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,19 +3,13 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
-import os.path
 import random
-import re
 import time
 import uuid
-import zlib
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import (
-    ExtractorError,
-    url_basename,
-)
+from ..utils import ExtractorError
 
 
 class IqiyiIE(InfoExtractor):

From 3af1fac7b0f43778e44b3b86e0c74bf25fb6f489 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 09:51:59 +0100
Subject: [PATCH 0795/2145] [dcn] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dcn.py      | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/dcn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..cc0da81d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -110,6 +110,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
+from .dcn import DcnIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
new file mode 100644
index 000000000..5263def4c
--- /dev/null
+++ b/youtube_dl/extractor/dcn.py
@@ -0,0 +1,46 @@
+from .common import InfoExtractor
+
+class DcnIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
+    _TEST = {
+        'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
+        'info_dict':
+        {
+            'id': '17375',
+            'ext': 'm3u8',
+            'title': 'رحلة العمر : الحلقة 1',
+            'description': '"في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة1"',
+            'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
+            'duration': '2041'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id='+video_id,
+            video_id
+        )
+        title = json_data['title_ar'];
+        thumbnail = 'http://admin.mangomolo.com/analytics/'+json_data['img'];
+        duration = json_data['duration'];
+        description = json_data['description_ar'];
+        webpage = self._download_webpage(
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id='+json_data['id']+'&user_id='+json_data['user_id']+'&countries=Q0M=&w=100%&h=100%&filter=DENY&signature='+json_data['signature'],
+            video_id
+        )
+        m3u8_url = self._html_search_regex(
+            r'file: "(?P<m3u8_url>.*?)"',
+            webpage,
+            'm3u8_url',
+            group='m3u8_url'
+        )
+        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'description': description,
+            'formats': formats,
+        }

From 9d681c2bb3b75a666b76d8e346ffab66b65f9132 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 10:00:24 +0100
Subject: [PATCH 0796/2145] remove unnecessary group name

---
 youtube_dl/extractor/dcn.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 5263def4c..f76ebda9e 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -30,10 +30,9 @@ class DcnIE(InfoExtractor):
             video_id
         )
         m3u8_url = self._html_search_regex(
-            r'file: "(?P<m3u8_url>.*?)"',
+            r'file:\s*"([^"]+)',
             webpage,
-            'm3u8_url',
-            group='m3u8_url'
+            'm3u8_url'
         )
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
         return {

From 36068ae019138710a3023334db2241ec815e2fe6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 03:54:59 +0800
Subject: [PATCH 0797/2145] [iqiyi] Comment out some MD5 sums

The value is different on Travis CI server
---
 youtube_dl/extractor/iqiyi.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 7caaa1183..d93b0867d 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -33,56 +33,48 @@ class IqiyiIE(InfoExtractor):
             'title': '名侦探柯南第752集',
         },
         'playlist': [{
-            'md5': '7e49376fecaffa115d951634917fe105',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '41b75ba13bb7ac0e411131f92bc4f6ca',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '0cee1dd0a3d46a83e71e2badeae2aab0',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '4f8ad72373b0c491b582e7c196b0b1f9',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': 'd89ad028bcfad282918e8098e811711d',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '9cb1e5c95da25dff0660c32ae50903b7',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '155116e0ff1867bbc9b98df294faabc9',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '53f5db77622ae14fa493ed2a278a082b',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
                 'ext': 'f4v',

From 114ed20e6435f9a68eb757f2d6e26e7ea941afbf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 22:10:57 +0100
Subject: [PATCH 0798/2145] [shahid] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/shahid.py   | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/shahid.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..6c1d4f08a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -502,6 +502,7 @@ from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
+from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sina import SinaIE
diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
new file mode 100644
index 000000000..d492070da
--- /dev/null
+++ b/youtube_dl/extractor/shahid.py
@@ -0,0 +1,56 @@
+from .common import InfoExtractor
+from ..utils import get_element_by_id
+
+class ShahidIE(InfoExtractor):
+    _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
+    _TESTS = [
+        {
+            'url': 'https://shahid.mbc.net/ar/episode/108084/%D8%AE%D9%88%D8%A7%D8%B7%D8%B1-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-11-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+            'info_dict': {
+                'id': '108084',
+                'ext': 'm3u8',
+                'title': 'بسم الله',
+                'description': 'بسم الله'
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
+        },
+        {
+            #shahid plus subscriber only
+            'url': 'https://shahid.mbc.net/ar/series/90497/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011.html',
+            'only_matching': True
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        json_data = self._parse_json(
+            get_element_by_id('jsonld', webpage),
+            video_id
+        )
+        title = json_data['name']
+        thumbnail = json_data['image']
+        categories = json_data['genre']
+        description = json_data['description']
+        player_json_data = self._download_json(
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-'+video_id+'.type-player.html',
+            video_id
+        )['data']
+        if 'url' in player_json_data:
+            m3u8_url = player_json_data['url']
+        else:
+            for error in json_data['error'].values():
+                self.report_warning(error)
+            return
+        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'description': description,
+            'formats': formats,
+        }

From 02c126a7c2453b101505e3c7b8209e49e36fcd71 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 22:55:40 +0100
Subject: [PATCH 0799/2145] [shahid] raise ExtractorError instead of warning

---
 youtube_dl/extractor/shahid.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index d492070da..6d76ef590 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,5 +1,8 @@
 from .common import InfoExtractor
-from ..utils import get_element_by_id
+from ..utils import (
+    get_element_by_id,
+    ExtractorError,
+}
 
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
@@ -43,7 +46,7 @@ class ShahidIE(InfoExtractor):
             m3u8_url = player_json_data['url']
         else:
             for error in json_data['error'].values():
-                self.report_warning(error)
+                raise ExtractorError(error)
             return
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
         return {

From 84c0ed50a5491146a8a6b489c530c2335c023f8f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 23:10:20 +0100
Subject: [PATCH 0800/2145] [shahid] fix backets closing

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 6d76ef590..9846c052c 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -2,7 +2,7 @@ from .common import InfoExtractor
 from ..utils import (
     get_element_by_id,
     ExtractorError,
-}
+)
 
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'

From 3c07a729a652a25caf9aac6143a19d040ecf9a15 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 23:16:57 +0100
Subject: [PATCH 0801/2145] [shahid] don't fail on missing info

---
 youtube_dl/extractor/shahid.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 9846c052c..04de866f4 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -35,9 +35,9 @@ class ShahidIE(InfoExtractor):
             video_id
         )
         title = json_data['name']
-        thumbnail = json_data['image']
-        categories = json_data['genre']
-        description = json_data['description']
+        thumbnail = json_data.get('image')
+        categories = json_data.get('genre')
+        description = json_data.get('description')
         player_json_data = self._download_json(
             'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-'+video_id+'.type-player.html',
             video_id

From e89d7e30294d3c5a36e6af5dd730ed543934d40e Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 03:34:22 +0200
Subject: [PATCH 0802/2145] [tagesschau] add support for more video types
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I found that currently only tagesschau videos are played. There are some
more shows hosted on tagesschau.de (see [0] for example) which are
easily playable by adjusting the regex. So this patch adds support for:
* tagesthemen
* tagesschau vor 20 Jahren
* tagesschau (mit Gebärdensprache)
* nachtmagazin

Note that some videos don't provide a description, so in order for the
tests to succeed, an ExtractorError needs to get caught.

[0]: http://www.tagesschau.de/multimedia/video/videoarchiv2~_date-20150714.html
---
 youtube_dl/extractor/tagesschau.py | 55 +++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index bfe07b024..2a2aafca0 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize
+from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -30,6 +30,46 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
+        'md5': '90757268b49ef56deae90c7b48928d58',
+        'info_dict': {
+            'id': '3771',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
+        'md5': '6e3ebdc75e8d67da966a8d06721eda71',
+        'info_dict': {
+            'id': '3827',
+            'ext': 'mp4',
+            'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66',
+            'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
+        'md5': '8a8875a568f0a5ae5ceef93c501a225f',
+        'info_dict': {
+            'id': '3475',
+            'ext': 'mp4',
+            'description': 'md5:ed149f5649cda3dac86813a9d777e131',
+            'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
+        'md5': 'be4d6f0421f2acd8abe25ea29f6f015b',
+        'info_dict': {
+            'id': '959',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {
@@ -102,9 +142,14 @@ class TagesschauIE(InfoExtractor):
             thumbnail_fn = self._search_regex(
                 r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
                 webpage, 'thumbnail', fatal=False)
-            description = self._html_search_regex(
-                r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', fatal=False)
+            # there are some videos without description
+            description = ""
+            try:
+                description = self._html_search_regex(
+                    r'(?s)<p class="teasertext">(.*?)</p>',
+                    webpage, 'description', fatal=False)
+            except ExtractorError:
+                pass
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From c51bc70e0feec42664c57228cc4b395a3a629420 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 03:52:54 +0200
Subject: [PATCH 0803/2145] [tagesschau] fix test which links to nonexisting
 video

---
 youtube_dl/extractor/tagesschau.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index bfe07b024..cf1b37a75 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -11,13 +11,13 @@ class TagesschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
-        'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
-        'md5': 'bcdeac2194fb296d599ce7929dfa4009',
+        'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
+        'md5': '917a228bc7df7850783bc47979673a09',
         'info_dict': {
-            'id': '1399128',
+            'id': '102143',
             'ext': 'mp4',
-            'title': 'Harald Range, Generalbundesanwalt, zu den Ermittlungen',
-            'description': 'md5:69da3c61275b426426d711bde96463ab',
+            'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
+            'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
             'thumbnail': 're:^http:.*\.jpg$',
         },
     }, {

From 3c6ae8b59ed2517891ff19507e4cef2322ef9eb4 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 04:27:58 +0200
Subject: [PATCH 0804/2145] [tagesschau] add support for Bericht aus Berlin
 videos

---
 youtube_dl/extractor/tagesschau.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 2a2aafca0..682f8df8f 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -70,6 +70,16 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
+        'md5': '42e3757018d9908581481a80cc1806da',
+        'info_dict': {
+            'id': '3299',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {

From 726adc43ec184b6d7e05cdbc197aa02c53b32347 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 05:09:29 +0200
Subject: [PATCH 0805/2145] [tagesschau] set description=None for empty
 descriptions

---
 youtube_dl/extractor/tagesschau.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 682f8df8f..4a755c657 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -36,7 +36,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '3771',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -66,7 +66,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '959',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -76,7 +76,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '3299',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -154,12 +154,9 @@ class TagesschauIE(InfoExtractor):
                 webpage, 'thumbnail', fatal=False)
             # there are some videos without description
             description = ""
-            try:
-                description = self._html_search_regex(
-                    r'(?s)<p class="teasertext">(.*?)</p>',
-                    webpage, 'description', fatal=False)
-            except ExtractorError:
-                pass
+            description = self._html_search_regex(
+                r'(?s)<p class="teasertext">(.*?)</p>',
+                webpage, 'description', fatal=False, default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From 4951c9f8214153d84033a77a4ef798ae29e1466a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 11:16:29 +0800
Subject: [PATCH 0806/2145] Credit @sceext2 for fixing iQiyi extractor (#6266)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 531ec5767..4fd65f46f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -132,3 +132,4 @@ George Brighton
 Remita Amine
 Aurélio A. Heckert
 Bernhard Minks
+sceext

From c2d1be8981c0f6b91635c32a3163e01b91d5db54 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 11:20:05 +0800
Subject: [PATCH 0807/2145] [iqiyi] Add skip_download to reduce network traffic

MD5 checksums are commented out in
36068ae019138710a3023334db2241ec815e2fe6, and actual downloads are not
necessary anymore.
---
 youtube_dl/extractor/iqiyi.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d93b0867d..afb7f4e61 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -81,6 +81,9 @@ class IqiyiIE(InfoExtractor):
                 'title': '名侦探柯南第752集',
             },
         }],
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _FORMATS_MAP = [

From 0215103e92c7a40aa0cea319e72dcf49f02053fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:13:27 +0200
Subject: [PATCH 0808/2145] [francetv] Remove unused import

---
 youtube_dl/extractor/francetv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d4f98ca16..477e22793 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -14,7 +14,6 @@ from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
-    float_or_none,
     parse_duration,
     determine_ext,
 )

From 8a7a2089050957c942bde0b9ab70963b2dea8f35 Mon Sep 17 00:00:00 2001
From: slangangular <slangangular@infrablue.org>
Date: Sun, 12 Jul 2015 10:30:51 +0200
Subject: [PATCH 0809/2145] [sportschau] Add support for sportschau.de

Closes #6199.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/sportschau.py | 43 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..d70de7690 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -552,6 +552,7 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
+from .sportschau import SportschauIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
new file mode 100644
index 000000000..ff30e1f0e
--- /dev/null
+++ b/youtube_dl/extractor/sportschau.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SportschauIE(InfoExtractor):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
+        'md5': 'a6ef460ab9f4089b079832e06d554cec',
+        'info_dict': {
+            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
+            'ext': 'mp4',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        ext = '-mc_defaultQuality-h.json'
+        json_url = url[:-5] + ext
+
+        json = self._download_json(json_url, video_id)
+        thumb_url = json['_previewImage']
+
+        m3u8_url = json['_mediaArray'][1]['_mediaStreamArray'][0]['_stream'][0]
+        m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        desc = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': m3u8_formats,
+            'description': desc,
+            'thumbnail': thumb_url,
+        }

From 8b61bfd6389b62f054cdf9dcb3436395c82a8e28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:21:18 +0200
Subject: [PATCH 0810/2145] [sportschau] skip download in test

---
 youtube_dl/extractor/sportschau.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
index ff30e1f0e..28797266f 100644
--- a/youtube_dl/extractor/sportschau.py
+++ b/youtube_dl/extractor/sportschau.py
@@ -9,14 +9,17 @@ class SportschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
     _TEST = {
         'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'md5': 'a6ef460ab9f4089b079832e06d554cec',
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
             'ext': 'mp4',
             'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 32470bf619d31605dc9c51ad107839a097f829f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:24:19 +0200
Subject: [PATCH 0811/2145] [sportschau] Improve title extraction

The html '<title>' ends with '- sportschau.de', which shouldn't be part of the title.
---
 youtube_dl/extractor/sportschau.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
index 28797266f..bf9b075db 100644
--- a/youtube_dl/extractor/sportschau.py
+++ b/youtube_dl/extractor/sportschau.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import get_element_by_attribute
 
 
 class SportschauIE(InfoExtractor):
@@ -12,7 +13,7 @@ class SportschauIE(InfoExtractor):
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
             'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
         },
@@ -34,7 +35,7 @@ class SportschauIE(InfoExtractor):
         m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        title = get_element_by_attribute('class', 'headline', webpage)
         desc = self._html_search_meta('description', webpage)
 
         return {

From 1dc31c2786b34f833acc5fc646afcf992f71444d Mon Sep 17 00:00:00 2001
From: Zach Bruggeman <mail@bruggie.com>
Date: Fri, 10 Jul 2015 19:27:48 -0700
Subject: [PATCH 0812/2145] [appleconnect] Add new extractor (fixes #6189)

Closes #6190.
---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/appleconnect.py | 50 ++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/appleconnect.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d70de7690..5033d67ed 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,6 +19,7 @@ from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
+from .appleconnect import AppleConnectIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
 from .ard import ARDIE, ARDMediathekIE
diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dl/extractor/appleconnect.py
new file mode 100644
index 000000000..ea7a70393
--- /dev/null
+++ b/youtube_dl/extractor/appleconnect.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    str_to_int,
+    ExtractorError
+)
+
+
+class AppleConnectIE(InfoExtractor):
+    _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
+    _TEST = {
+        'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
+        'md5': '10d0f2799111df4cb1c924520ca78f98',
+        'info_dict': {
+            'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
+            'ext': 'm4v',
+            'title': 'Energy',
+            'uploader': 'Drake',
+            'thumbnail': 'http://is5.mzstatic.com/image/thumb/Video5/v4/78/61/c5/7861c5fa-ad6d-294b-1464-cf7605b911d6/source/1920x1080sr.jpg',
+            'upload_date': '20150710',
+            'timestamp': 1436545535,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        try:
+            video_json = self._html_search_regex(
+                r'class="auc-video-data">(\{.*?\})', webpage, 'json')
+        except ExtractorError:
+            raise ExtractorError('This post doesn\'t contain a video', expected=True)
+
+        video_data = self._parse_json(video_json, video_id)
+        timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
+        like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count'))
+
+        return {
+            'id': video_id,
+            'url': video_data['sslSrc'],
+            'title': video_data['title'],
+            'description': video_data['description'],
+            'uploader': video_data['artistName'],
+            'thumbnail': video_data['artworkUrl'],
+            'timestamp': timestamp,
+            'like_count': like_count,
+        }

From ecdbe09e1017fdd771720f9954f8427b9084bbf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:45:49 +0600
Subject: [PATCH 0813/2145] [francetv] Fix f4m extraction

---
 youtube_dl/extractor/francetv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 477e22793..624895ff9 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -58,9 +58,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     # See https://github.com/rg3/youtube-dl/issues/3963
                     # m3u8 urls work fine
                     continue
-                video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
-                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url_parsed.path,
+                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))

From 789a12aaafe7036a155435d9c93d062309f547b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:50:25 +0600
Subject: [PATCH 0814/2145] [francetv] Restore support for jt videos

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 624895ff9..11db6d921 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -164,7 +164,8 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                             (?:www\.)?france[2345o]\.fr/
                                 (?:
                                     emissions/[^/]+/(?:videos|diffusions)?|
-                                    videos
+                                    videos|
+                                    jt
                                 )
                             /|
                             embed\.francetv\.fr/\?ue=

From 632cbb8efa32ef704219a769972fdeae04577ab8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:51:06 +0600
Subject: [PATCH 0815/2145] [francetv] Fix f4m extraction completely

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 11db6d921..746ab9d84 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -62,7 +62,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
-                    formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))
+                    formats.extend(self._extract_f4m_formats(
+                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, 1, format_id))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
             elif video_url.startswith('rtmp'):

From 3bc9fb58892d0ca3dfc044b6e0ef65426ecd8c1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:53:09 +0600
Subject: [PATCH 0816/2145] [francetv] Update jt test

---
 youtube_dl/extractor/francetv.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 746ab9d84..f5aeab110 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -229,15 +229,16 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         },
         # franceo
         {
-            'url': 'http://www.franceo.fr/jt/info-afrique/04-12-2013',
-            'md5': '52f0bfe202848b15915a2f39aaa8981b',
+            'url': 'http://www.franceo.fr/jt/info-soir/18-07-2015',
+            'md5': '47d5816d3b24351cdce512ad7ab31da8',
             'info_dict': {
-                'id': '108634970',
+                'id': '125377621',
                 'ext': 'flv',
-                'title': 'Infô Afrique',
-                'description': 'md5:ebf346da789428841bee0fd2a935ea55',
-                'upload_date': '20140915',
-                'timestamp': 1410822000,
+                'title': 'Infô soir',
+                'description': 'md5:01b8c6915a3d93d8bbbd692651714309',
+                'upload_date': '20150718',
+                'timestamp': 1437241200,
+                'duration': 414,
             },
         },
         {

From ac4b8df5e47a113da5bb66fc975f938bc176b58c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:53:54 +0600
Subject: [PATCH 0817/2145] [francetv] Fix embed test

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index f5aeab110..08b124805 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -248,7 +248,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'id': 'EV_30231',
                 'ext': 'flv',
                 'title': 'Alcaline, le concert avec Calogero',
-                'description': 'md5:',
+                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
                 'upload_date': '20150226',
                 'timestamp': 1424989860,
                 'duration': 5400,

From 1d18e26ecad636cb88ee9710807e9583eec98cd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:54:12 +0600
Subject: [PATCH 0818/2145] [francetv] Remove unused import

---
 youtube_dl/extractor/francetv.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 08b124805..706ed9c99 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -6,10 +6,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     clean_html,
     ExtractorError,

From 7a4a945f138078b95f02e8cec1e38123d66d7f09 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 11:31:38 -0500
Subject: [PATCH 0819/2145] fix/support news.vice.com

---
 youtube_dl/extractor/vice.py | 40 ++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 04e2b0ba7..dfb621468 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,20 +7,34 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(.+?\.)?vice\.com/.*?/(?P<name>.+)'
 
-    _TEST = {
-        'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
-        'info_dict': {
-            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
-            'ext': 'mp4',
-            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-        },
-        'params': {
-            # Requires ffmpeg (m3u8 manifest)
-            'skip_download': True,
-        },
-    }
+    _TESTS = [
+        {
+            'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
+            'info_dict': {
+                'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+                'ext': 'mp4',
+                'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+            },
+            'params': {
+                # Requires ffmpeg (m3u8 manifest)
+                'skip_download': True,
+            },
+        }, {
+            'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+            'info_dict': {
+                'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
+                'ext': 'mp4',
+                'title': 'VICE News - Inside the Monkey Lab',
+                'description': 'md5:1f660d467d3515f29d11e5ef742a4b82',
+            },
+            'params': {
+                # Requires ffmpeg (m3u8 manifest)
+                'skip_download': True,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From a5dd9a0c5dcf6aace1b7af87acb982480d606753 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 11:47:58 -0500
Subject: [PATCH 0820/2145] pbs: fix vague 'Full Episode' titles; prepend name
 of show

Many videos are given the title 'Full Episode' or 'Episode x', etc;
in the info json 'title' is the episode title (for oneshots this is
'Full Episode') while the program name is 'program':{'title'. This
changes the title from '(episode title)' to
 '(program name) - (episode title)'

The following urls demonstrate useless titles

    http://video.pbs.org/video/2365367186/
    http://video.pbs.org/video/2365519307/
    http://video.pbs.org/video/2365527039/
    http://video.pbs.org/video/2365530605/

Before change

    Full Episode
    Episode 5 | Preview
    Season 3 | Episode 4 Preview
    July 17, 2015

After change

    To Catch A Comet - Full Episode
    The Crimson Field - Episode 5 | Preview
    Last Tango in Halifax - Season 3 | Episode 4 Preview
    Charlie Rose The Week - July 17, 2015
---
 youtube_dl/extractor/pbs.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index fec5d65ad..cccff270b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -32,7 +32,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365006249',
                 'ext': 'mp4',
-                'title': 'A More Perfect Union',
+                'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
                 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
                 'duration': 3190,
             },
@@ -46,7 +46,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365297690',
                 'ext': 'mp4',
-                'title': 'Losing Iraq',
+                'title': 'FRONTLINE - Losing Iraq',
                 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
                 'duration': 5050,
             },
@@ -60,7 +60,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2201174722',
                 'ext': 'mp4',
-                'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
+                'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
                 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
                 'duration': 801,
             },
@@ -72,7 +72,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297708',
                 'ext': 'mp4',
                 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
-                'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+                'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -88,7 +88,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'killer-typhoon',
                 'ext': 'mp4',
                 'description': 'md5:c741d14e979fc53228c575894094f157',
-                'title': 'Killer Typhoon',
+                'title': 'NOVA - Killer Typhoon',
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
@@ -110,7 +110,7 @@ class PBSIE(InfoExtractor):
                 'id': '2280706814',
                 'display_id': 'player',
                 'ext': 'mp4',
-                'title': 'Death and the Civil War',
+                'title': 'American Experience - Death and the Civil War',
                 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
                 'duration': 6705,
                 'thumbnail': 're:^https?://.*\.jpg$',
@@ -118,6 +118,21 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            'url': 'http://video.pbs.org/video/2365367186/',
+            'info_dict': {
+                'id': '2365367186',
+                'display_id': '2365367186',
+                'ext': 'mp4',
+                'title': 'To Catch A Comet - Full Episode',
+                'description': 'On November 12, 2014, billions of kilometers from Earth, spacecraft orbiter Rosetta and lander Philae did what no other had dared to attempt \u2014 land on the volatile surface of a comet as it zooms around the sun at 67,000 km/hr. The European Space Agency hopes this mission can help peer into our past and unlock secrets of our origins.',
+                'duration': 3342,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
 
@@ -232,6 +247,12 @@ class PBSIE(InfoExtractor):
                 'url': closed_captions_url,
             }]
 
+        # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
+        # 'Full Episode', 'Episode 5', etc. prepend program->title
+        alt_title = info.get('program', {}).get('title', '')
+        if alt_title != '':
+            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
+
         return {
             'id': video_id,
             'display_id': display_id,

From e37c92ec6d137cd2d09c5f13e9a261c4fae552dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 23:59:50 +0600
Subject: [PATCH 0821/2145] [ard] Extract all formats

---
 youtube_dl/extractor/ard.py | 125 ++++++++++++++++++++++++++----------
 1 file changed, 91 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6a35ea463..55f940d57 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -35,6 +35,87 @@ class ARDMediathekIE(InfoExtractor):
         'skip': 'Blocked outside of Germany',
     }]
 
+    def _extract_media_info(self, media_info_url, webpage, video_id):
+        media_info = self._download_json(
+            media_info_url, video_id, 'Downloading media JSON')
+
+        formats = self._extract_formats(media_info, video_id)
+
+        if not formats:
+            if '"fsk"' in webpage:
+                raise ExtractorError(
+                    'This video is only available after 20:00', expected=True)
+            elif media_info.get('_geoblocked'):
+                raise ExtractorError('This video is not available due to geo restriction', expected=True)
+
+        self._sort_formats(formats)
+
+        duration = int_or_none(media_info.get('_duration'))
+        thumbnail = media_info.get('_previewImage')
+
+        subtitles = {}
+        subtitle_url = media_info.get('_subtitleUrl')
+        if subtitle_url:
+            subtitles['de'] = [{
+                'ext': 'srt',
+                'url': subtitle_url,
+            }]
+
+        return {
+            'id': video_id,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _extract_formats(self, media_info, video_id):
+        type_ = media_info.get('_type')
+        media_array = media_info.get('_mediaArray', [])
+        formats = []
+        for num, media in enumerate(media_array):
+            for stream in media.get('_mediaStreamArray', []):
+                stream_urls = stream.get('_stream')
+                if not stream_urls:
+                    continue
+                if not isinstance(stream_urls, list):
+                    stream_urls = [stream_urls]
+                quality = stream.get('_quality')
+                server = stream.get('_server')
+                for stream_url in stream_urls:
+                    ext = determine_ext(stream_url)
+                    if ext == 'f4m':
+                        formats.extend(self._extract_f4m_formats(
+                            stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
+                            video_id, preference=-1, f4m_id='hds'))
+                    elif ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls'))
+                    else:
+                        if server and server.startswith('rtmp'):
+                            f = {
+                                'url': server,
+                                'play_path': stream_url,
+                                'format_id': 'a%s-rtmp-%s' % (num, quality),
+                            }
+                        elif stream_url.startswith('http'):
+                            f = {
+                                'url': stream_url,
+                                'format_id': 'a%s-%s-%s' % (num, ext, quality)
+                            }
+                        else:
+                            continue
+                        m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', stream_url)
+                        if m:
+                            f.update({
+                                'width': int(m.group('width')),
+                                'height': int(m.group('height')),
+                            })
+                        if type_ == 'audio':
+                            f['vcodec'] = 'none'
+                        formats.append(f)
+        return formats
+
     def _real_extract(self, url):
         # determine video id from url
         m = re.match(self._VALID_URL, url)
@@ -92,46 +173,22 @@ class ARDMediathekIE(InfoExtractor):
                     'format_id': fid,
                     'url': furl,
                 })
+            self._sort_formats(formats)
+            info = {
+                'formats': formats,
+            }
         else:  # request JSON file
-            media_info = self._download_json(
-                'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
-            # The second element of the _mediaArray contains the standard http urls
-            streams = media_info['_mediaArray'][1]['_mediaStreamArray']
-            if not streams:
-                if '"fsk"' in webpage:
-                    raise ExtractorError('This video is only available after 20:00')
+            info = self._extract_media_info(
+                'http://www.ardmediathek.de/play/media/%s' % video_id, webpage, video_id)
 
-            formats = []
-            for s in streams:
-                if type(s['_stream']) == list:
-                    for index, url in enumerate(s['_stream'][::-1]):
-                        quality = s['_quality'] + index
-                        formats.append({
-                            'quality': quality,
-                            'url': url,
-                            'format_id': '%s-%s' % (determine_ext(url), quality)
-                        })
-                    continue
-
-                format = {
-                    'quality': s['_quality'],
-                    'url': s['_stream'],
-                }
-
-                format['format_id'] = '%s-%s' % (
-                    determine_ext(format['url']), format['quality'])
-
-                formats.append(format)
-
-        self._sort_formats(formats)
-
-        return {
+        info.update({
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': formats,
             'thumbnail': thumbnail,
-        }
+        })
+
+        return info
 
 
 class ARDIE(InfoExtractor):

From 3bf8c316a663741ab806f48cf9121209aba88b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:01:22 +0600
Subject: [PATCH 0822/2145] [sportschau] Reimplement in terms of ard extractor

---
 youtube_dl/extractor/__init__.py   |  7 +++--
 youtube_dl/extractor/ard.py        | 39 +++++++++++++++++++++++++
 youtube_dl/extractor/sportschau.py | 47 ------------------------------
 3 files changed, 44 insertions(+), 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5033d67ed..50da08830 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -22,7 +22,11 @@ from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
-from .ard import ARDIE, ARDMediathekIE
+from .ard import (
+    ARDIE,
+    ARDMediathekIE,
+    SportschauIE,
+)
 from .arte import (
     ArteTvIE,
     ArteTVPlus7IE,
@@ -553,7 +557,6 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
-from .sportschau import SportschauIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 55f940d57..2c368d833 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -8,6 +8,7 @@ from .generic import GenericIE
 from ..utils import (
     determine_ext,
     ExtractorError,
+    get_element_by_attribute,
     qualities,
     int_or_none,
     parse_duration,
@@ -246,3 +247,41 @@ class ARDIE(InfoExtractor):
             'upload_date': upload_date,
             'thumbnail': thumbnail,
         }
+
+
+class SportschauIE(ARDMediathekIE):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
+        'info_dict': {
+            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
+            'ext': 'mp4',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        base_url = mobj.group('baseurl')
+
+        webpage = self._download_webpage(url, video_id)
+        title = get_element_by_attribute('class', 'headline', webpage)
+        description = self._html_search_meta('description', webpage, 'description')
+
+        info = self._extract_media_info(
+            base_url + '-mc_defaultQuality-h.json', webpage, video_id)
+
+        info.update({
+            'title': title,
+            'description': description,
+        })
+
+        return info
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
deleted file mode 100644
index bf9b075db..000000000
--- a/youtube_dl/extractor/sportschau.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import get_element_by_attribute
-
-
-class SportschauIE(InfoExtractor):
-    IE_NAME = 'Sportschau'
-    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
-    _TEST = {
-        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'info_dict': {
-            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
-            'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        ext = '-mc_defaultQuality-h.json'
-        json_url = url[:-5] + ext
-
-        json = self._download_json(json_url, video_id)
-        thumb_url = json['_previewImage']
-
-        m3u8_url = json['_mediaArray'][1]['_mediaStreamArray'][0]['_stream'][0]
-        m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
-
-        webpage = self._download_webpage(url, video_id)
-        title = get_element_by_attribute('class', 'headline', webpage)
-        desc = self._html_search_meta('description', webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': m3u8_formats,
-            'description': desc,
-            'thumbnail': thumb_url,
-        }

From 86b4e98ac6f57c3703cd2082a69351ece797111c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:05:10 +0600
Subject: [PATCH 0823/2145] [ard:mediathek] Add audio test

---
 youtube_dl/extractor/ard.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 2c368d833..fdb94af6b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -34,6 +34,17 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
         },
         'skip': 'Blocked outside of Germany',
+    }, {
+        # audio
+        'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
+        'md5': '219d94d8980b4f538c7fcb0865eb7f2c',
+        'info_dict': {
+            'id': '28488308',
+            'ext': 'mp3',
+            'title': 'Tod eines Fußballers',
+            'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
+            'duration': 3240,
+        },
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):
@@ -252,7 +263,7 @@ class ARDIE(InfoExtractor):
 class SportschauIE(ARDMediathekIE):
     IE_NAME = 'Sportschau'
     _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
@@ -265,7 +276,7 @@ class SportschauIE(ARDMediathekIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 769efa16afc1c9a2871543f69081f1bced361dc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:08:21 +0600
Subject: [PATCH 0824/2145] [ard:mediathek] Remove dead test

---
 youtube_dl/extractor/ard.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index fdb94af6b..c78475748 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -23,17 +23,18 @@ class ARDMediathekIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
 
     _TESTS = [{
-        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
+        'url': 'http://www.ardmediathek.de/tv/Dokumentation-und-Reportage/Ich-liebe-das-Leben-trotzdem/rbb-Fernsehen/Video?documentId=29582122&bcastId=3822114',
         'info_dict': {
-            'id': '22490580',
+            'id': '29582122',
             'ext': 'mp4',
-            'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
-            'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
+            'title': 'Ich liebe das Leben trotzdem',
+            'description': 'md5:45e4c225c72b27993314b31a84a5261c',
+            'duration': 4557,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-        'skip': 'Blocked outside of Germany',
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
@@ -45,6 +46,9 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
             'duration': 3240,
         },
+    }, {
+        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
+        'only_matching': True,
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):

From d719c6a5abf746c31f301ed1a1d8934715eaeb05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:13:39 +0600
Subject: [PATCH 0825/2145] [ard:mediathek] Add test for direct mp4

---
 youtube_dl/extractor/ard.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index c78475748..6f465789b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -35,6 +35,16 @@ class ARDMediathekIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.ardmediathek.de/tv/Tatort/Tatort-Scheinwelten-H%C3%B6rfassung-Video/Das-Erste/Video?documentId=29522730&bcastId=602916',
+        'md5': 'f4d98b10759ac06c0072bbcd1f0b9e3e',
+        'info_dict': {
+            'id': '29522730',
+            'ext': 'mp4',
+            'title': 'Tatort: Scheinwelten - Hörfassung (Video tgl. ab 20 Uhr)',
+            'description': 'md5:196392e79876d0ac94c94e8cdb2875f1',
+            'duration': 5252,
+        },
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',

From 772acaf31f4614d390a445fe4c0d4e533c165774 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:39:55 +0600
Subject: [PATCH 0826/2145] [vice] Do not capture unused groups in _VALID_URL

---
 youtube_dl/extractor/vice.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index dfb621468..e3ad4ef54 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(.+?\.)?vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/.*?/(?P<name>.+)'
 
     _TESTS = [
         {

From 65c2b21df15053581cac21690605d1f7ac834083 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:40:43 +0600
Subject: [PATCH 0827/2145] [vice] Make test only matching

---
 youtube_dl/extractor/vice.py | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index e3ad4ef54..ae1faca1c 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -23,16 +23,7 @@ class ViceIE(InfoExtractor):
             },
         }, {
             'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-            'info_dict': {
-                'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
-                'ext': 'mp4',
-                'title': 'VICE News - Inside the Monkey Lab',
-                'description': 'md5:1f660d467d3515f29d11e5ef742a4b82',
-            },
-            'params': {
-                # Requires ffmpeg (m3u8 manifest)
-                'skip_download': True,
-            },
+            'only_matching': True,
         }
     ]
 

From 18ae46ad4bcf4d427984e85a9feaf647c3f80737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:42:25 +0600
Subject: [PATCH 0828/2145] [vice] Modernize

---
 youtube_dl/extractor/vice.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index ae1faca1c..01af7a995 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -1,5 +1,4 @@
 from __future__ import unicode_literals
-import re
 
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
@@ -7,7 +6,7 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)+(?P<id>.+)'
 
     _TESTS = [
         {
@@ -28,9 +27,8 @@ class ViceIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
         try:
             embed_code = self._search_regex(
                 r'embedCode=([^&\'"]+)', webpage,

From 0eacd2aaae7d4150d9cf4e1dd8ffc2ebc7ab030b Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 13:59:12 -0500
Subject: [PATCH 0829/2145] less clunky if statement

---
 youtube_dl/extractor/pbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cccff270b..ccbe2a9f3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -249,8 +249,8 @@ class PBSIE(InfoExtractor):
 
         # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
         # 'Full Episode', 'Episode 5', etc. prepend program->title
-        alt_title = info.get('program', {}).get('title', '')
-        if alt_title != '':
+        alt_title = info.get('program', {}).get('title')
+        if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
 
         return {

From 308c505c3d66b2a11df8c1a5cd7e0fdaa694b585 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 01:03:43 +0600
Subject: [PATCH 0830/2145] [francetv] Improve _VALID_URL

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 706ed9c99..75723c00d 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -161,7 +161,8 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                         (?:
                             (?:www\.)?france[2345o]\.fr/
                                 (?:
-                                    emissions/[^/]+/(?:videos|diffusions)?|
+                                    emissions/[^/]+/(?:videos|diffusions)|
+                                    emission/[^/]+|
                                     videos|
                                     jt
                                 )

From faa1f83ab481a1f456db13d3ead7f0f22908aab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 02:16:07 +0600
Subject: [PATCH 0831/2145] [twitch:stream] Improve _VALID_URL (Closes #6294)

---
 youtube_dl/extractor/twitch.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 948c8ce39..73ce335b7 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -310,9 +310,9 @@ class TwitchBookmarksIE(TwitchPlaylistBaseIE):
 
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',
         'info_dict': {
             'id': '12772022048',
@@ -331,7 +331,10 @@ class TwitchStreamIE(TwitchBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)

From a1b85269a49404cab1f549d7c59352fe48dfbf19 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:49:53 +0800
Subject: [PATCH 0832/2145] [extractor/generic] Support vid.me embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc24a8a8b..cd133a10c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1196,6 +1196,12 @@ class GenericIE(InfoExtractor):
         if vimeo_url is not None:
             return self.url_result(vimeo_url)
 
+        vid_me_embed_url = self._search_regex(
+            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
+            webpage, 'vid.me embed', default=None)
+        if vid_me_embed_url is not None:
+            return self.url_result(vid_me_embed_url, 'Vidme')
+
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
             (?:

From c909e5820e94ecfefd9a2eb41b1beaa684c530c4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:51:25 +0800
Subject: [PATCH 0833/2145] [tumblr] Delegate to GenericIE for non-tumblr
 videos

Fixes #6162
---
 youtube_dl/extractor/tumblr.py | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 9ead13a91..772f94f7d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,8 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .pornhub import PornHubIE
-from .vimeo import VimeoIE
 
 
 class TumblrIE(InfoExtractor):
@@ -60,25 +58,14 @@ class TumblrIE(InfoExtractor):
         blog = m_url.group('blog_name')
 
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
-        webpage = self._download_webpage(url, video_id)
-
-        vid_me_embed_url = self._search_regex(
-            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
-            webpage, 'vid.me embed', default=None)
-        if vid_me_embed_url is not None:
-            return self.url_result(vid_me_embed_url, 'Vidme')
-
-        pornhub_url = PornHubIE._extract_url(webpage)
-        if pornhub_url:
-            return self.url_result(pornhub_url, 'PornHub')
-
-        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
-        if vimeo_url:
-            return self.url_result(vimeo_url, 'Vimeo')
+        webpage, urlh = self._download_webpage_handle(url, video_id)
 
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
-            webpage, 'iframe url')
+            webpage, 'iframe url', default=None)
+        if iframe_url is None:
+            return self.url_result(urlh.geturl(), 'Generic')
+
         iframe = self._download_webpage(iframe_url, video_id)
         video_url = self._search_regex(r'<source src="([^"]+)"',
                                        iframe, 'video url')

From 3f302bca8cf29f618d1f1c2fb7bc4ea9538af805 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:54:47 +0800
Subject: [PATCH 0834/2145] [tumblr] Improve downloading notes

---
 youtube_dl/extractor/tumblr.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 772f94f7d..3d3b635e4 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -66,7 +66,8 @@ class TumblrIE(InfoExtractor):
         if iframe_url is None:
             return self.url_result(urlh.geturl(), 'Generic')
 
-        iframe = self._download_webpage(iframe_url, video_id)
+        iframe = self._download_webpage(iframe_url, video_id,
+                                        'Downloading iframe page')
         video_url = self._search_regex(r'<source src="([^"]+)"',
                                        iframe, 'video url')
 

From 17ee98e1a5a6897ed0d9c52d9257b45ed9c3f20a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 19:48:50 +0200
Subject: [PATCH 0835/2145] [youtube] Fix upload_date in test

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9a08924ef..f73032abd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -543,7 +543,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'CsmdDsKjzN8',
                 'ext': 'mp4',
-                'upload_date': '20150510',
+                'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
                 'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',

From 2fe1ff85829244f6498dde8c85ecdc1415f5ad7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:14:20 +0200
Subject: [PATCH 0836/2145] [youtube] Set 'is_live'

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f73032abd..3d8b31f98 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -915,6 +915,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Get video info
         embed_webpage = None
+        is_live = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -947,6 +948,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
+                if args.get('livestream') == '1' or args.get('live_playback') == 1:
+                    is_live = True
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1251,6 +1254,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'dislike_count': dislike_count,
             'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
+            'is_live': is_live,
         }
 
 

From 8250c32f49fed0986bd514d1435e00a0182e8314 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:25:53 +0200
Subject: [PATCH 0837/2145] [YoutubeDL] don't default to
 'bestvideo+bestaudio/best' for live videos

Doesn't work currently.
---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 00af78e06..702a6ad50 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1104,7 +1104,8 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    info_dict['extractor'] in ['youtube', 'ted']):
+                    info_dict['extractor'] in ['youtube', 'ted'] and
+                    not info_dict.get('is_live')):
                 merger = FFmpegMergerPP(self)
                 if merger.available and merger.can_merge():
                     req_format_list.append('bestvideo+bestaudio')

From 7c80519cbf7e3daa029239fbbd147652817877f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 21:10:28 +0200
Subject: [PATCH 0838/2145] [youtube] Extract start_time

From the 't=*' in the url.
Currently youtube-dl doesn't use the value, but it was requested for the mpv plugin.
---
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b9014fc23..9e8751877 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -183,6 +183,8 @@ class InfoExtractor(object):
                     ["Sports", "Berlin"]
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
+    start_time:     Time in seconds where the reproduction should start, as
+                    specified in the url.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d8b31f98..afbd34f4a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -19,6 +19,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
@@ -31,6 +32,7 @@ from ..utils import (
     get_element_by_id,
     int_or_none,
     orderedSet,
+    parse_duration,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -317,7 +319,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -329,6 +331,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'categories': ['Science & Technology'],
                 'like_count': int,
                 'dislike_count': int,
+                'start_time': 1,
             }
         },
         {
@@ -889,6 +892,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'http' if self._downloader.params.get('prefer_insecure', False)
             else 'https')
 
+        start_time = None
+        parsed_url = compat_urllib_parse_urlparse(url)
+        for component in [parsed_url.fragment, parsed_url.query]:
+            query = compat_parse_qs(component)
+            if 't' in query:
+                start_time = parse_duration(query['t'][0])
+                break
+
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
         if mobj:
@@ -1255,6 +1266,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
             'is_live': is_live,
+            'start_time': start_time,
         }
 
 

From f993afb26d20ac79891621494922887b3c8df02b Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Mon, 20 Jul 2015 23:10:29 +0200
Subject: [PATCH 0839/2145] README: fix a typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2cc89cdb..ac54d7b67 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
     --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8"
                                      if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
                                      download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
     --match-title REGEX              Download only matching titles (regex or caseless sub-string)

From 4eb59a6b1cd1c4ca5f7dd284f0c26a329bb8e973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jul 2015 09:11:23 +0600
Subject: [PATCH 0840/2145] [options] Fix a typo (#6307)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 85365d769..9016e3498 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -219,7 +219,7 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--playlist-items',
         dest='playlist_items', metavar='ITEM_SPEC', default=None,
-        help='Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
+        help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',

From ce9512b78bbd5f180f44670f6b78784a0be3b815 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 21 Jul 2015 17:20:54 +0200
Subject: [PATCH 0841/2145] release 2015.07.21

---
 docs/supportedsites.md | 6 ++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a84878026..73445137f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
+ - **AppleConnect**
  - **AppleDaily**: 臺灣蘋果日報
  - **AppleTrailers**
  - **archive.org**: archive.org videos
@@ -486,6 +487,7 @@
  - **SportBox**
  - **SportBoxEmbed**
  - **SportDeutschland**
+ - **Sportschau**
  - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
@@ -611,8 +613,8 @@
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
- - **vk.com**
- - **vk.com:user-videos**: vk.com:All of a user's videos
+ - **vk**: VK
+ - **vk:uservideos**: VK - User's Videos
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3ad7a2bc0..280afdd7f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.18'
+__version__ = '2015.07.21'

From 496ce6b3493757e806d857363955a2289410a4e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jul 2015 23:54:31 +0600
Subject: [PATCH 0842/2145] [snagfilms] Improve m3u8 extraction (Closes #6309)

---
 youtube_dl/extractor/snagfilms.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cf495f310..0e820d250 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -23,6 +23,15 @@ class SnagFilmsEmbedIE(InfoExtractor):
             'ext': 'mp4',
             'title': '#whilewewatch',
         }
+    }, {
+        # invalid labels, 360p is better that 480p
+        'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
+        'md5': '882fca19b9eb27ef865efeeaed376a48',
+        'info_dict': {
+            'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
+            'ext': 'mp4',
+            'title': 'Life in Limbo',
+        }
     }, {
         'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
         'only_matching': True,
@@ -52,14 +61,15 @@ class SnagFilmsEmbedIE(InfoExtractor):
             if not file_:
                 continue
             type_ = source.get('type')
-            format_id = source.get('label')
             ext = determine_ext(file_)
-            if any(_ == 'm3u8' for _ in (type_, ext)):
+            format_id = source.get('label') or ext
+            if all(_ == 'm3u8' for _ in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:
                 bitrate = int_or_none(self._search_regex(
-                    r'(\d+)kbps', file_, 'bitrate', default=None))
+                    [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
+                    file_, 'bitrate', default=None))
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]$', format_id, 'height', default=None))
                 formats.append({

From 28afa6e77a8fb95541ae9bead0ec0e57958cbfc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Jul 2015 20:50:02 +0200
Subject: [PATCH 0843/2145] [snagfilms] Don't use '_' as a variable that is
 used

---
 youtube_dl/extractor/snagfilms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 0e820d250..6977afb27 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -63,7 +63,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
             type_ = source.get('type')
             ext = determine_ext(file_)
             format_id = source.get('label') or ext
-            if all(_ == 'm3u8' for _ in (type_, ext)):
+            if all(v == 'm3u8' for v in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:

From b68a2613f856fbbf943abf8d6de6ffc6aff2e930 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 02:00:21 +0600
Subject: [PATCH 0844/2145] [viewster] Rewrite for new API (Closes #6317)

---
 youtube_dl/extractor/viewster.py | 218 ++++++++++++++++---------------
 1 file changed, 113 insertions(+), 105 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 1742e66f4..03c873608 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -1,129 +1,137 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class ViewsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
+    _VALID_URL = r'http://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
     _TESTS = [{
-        # movielink, paymethod=fre
-        'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
-        'playlist': [{
-            'md5': '8f9d94b282d80c42b378dffdbb11caf3',
-            'info_dict': {
-                'id': '1293-19341-000-movie',
-                'ext': 'flv',
-                'title': "'Hout' (Wood) - Movie",
-            },
-        }],
-        'info_dict': {
-            'id': '1293-19341-000',
-            'title': "'Hout' (Wood)",
-            'description': 'md5:925733185a9242ef96f436937683f33b',
-        }
-    }, {
-        # movielink, paymethod=adv
+        # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
-        'playlist': [{
-            'md5': '77a005453ca7396cbe3d35c9bea30aef',
-            'info_dict': {
-                'id': '1140-11855-000-movie',
-                'ext': 'flv',
-                'title': "THE LISTENING PROJECT - Movie",
-            },
-        }],
+        'md5': '14d3cfffe66d57b41ae2d9c873416f01',
         'info_dict': {
             'id': '1140-11855-000',
-            'title': "THE LISTENING PROJECT",
-            'description': 'md5:714421ae9957e112e672551094bf3b08',
-        }
+            'ext': 'flv',
+            'title': 'The listening Project',
+            'description': 'md5:bac720244afd1a8ea279864e67baa071',
+            'timestamp': 1214870400,
+            'upload_date': '20080701',
+            'duration': 4680,
+        },
     }, {
-        # direct links, no movielink
-        'url': 'http://www.viewster.com/movie/1198-56411-000/sinister/',
-        'playlist': [{
-            'md5': '0307b7eac6bfb21ab0577a71f6eebd8f',
-            'info_dict': {
-                'id': '1198-56411-000-trailer',
-                'ext': 'mp4',
-                'title': "Sinister - Trailer",
-            },
-        }, {
-            'md5': '80b9ee3ad69fb368f104cb5d9732ae95',
-            'info_dict': {
-                'id': '1198-56411-000-behind-scenes',
-                'ext': 'mp4',
-                'title': "Sinister - Behind Scenes",
-            },
-        }, {
-            'md5': '3b3ea897ecaa91fca57a8a94ac1b15c5',
-            'info_dict': {
-                'id': '1198-56411-000-scene-from-movie',
-                'ext': 'mp4',
-                'title': "Sinister - Scene from movie",
-            },
-        }],
+        # series episode, Type=Episode
+        'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
+        'md5': 'd5434c80fcfdb61651cc2199a88d6ba3',
         'info_dict': {
-            'id': '1198-56411-000',
-            'title': "Sinister",
-            'description': 'md5:014c40b0488848de9683566a42e33372',
-        }
+            'id': '1284-19427-001',
+            'ext': 'flv',
+            'title': 'The World and a Wall',
+            'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
+            'timestamp': 1428192000,
+            'upload_date': '20150405',
+            'duration': 1500,
+        },
+    }, {
+        # serie, Type=Serie
+        'url': 'http://www.viewster.com/serie/1303-19426-000/',
+        'info_dict': {
+            'id': '1303-19426-000',
+            'title': 'Is It Wrong to Try to Pick up Girls in a Dungeon?',
+            'description': 'md5:eeda9bef25b0d524b3a29a97804c2f11',
+        },
+        'playlist_count': 13,
+    }, {
+        # unfinished serie, no Type
+        'url': 'http://www.viewster.com/serie/1284-19427-000/baby-steps-season-2/',
+        'info_dict': {
+            'id': '1284-19427-000',
+            'title': 'Baby Steps—Season 2',
+            'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
+        },
+        'playlist_mincount': 16,
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
+    _AUTH_TOKEN = '/YqhSYsx8EaU9Bsta3ojlA=='
+
+    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
+        request = compat_urllib_request.Request(url)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+        request.add_header('Auth-token', self._AUTH_TOKEN)
+        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
-            'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
-        request.add_header('Accept', self._ACCEPT_HEADER)
+        info = self._download_json(
+            'https://public-api.viewster.com/search/%s' % video_id,
+            video_id, 'Downloading entry JSON')
 
-        movie = self._download_json(
-            request, video_id, 'Downloading movie metadata JSON')
+        entry_id = info.get('Id') or info['id']
 
-        title = movie.get('title') or movie['original_title']
-        description = movie.get('synopsis')
-        thumbnail = movie.get('large_artwork') or movie.get('artwork')
+        # unfinished serie has no Type
+        if info.get('Type') in ['Serie', None]:
+            episodes = self._download_json(
+                'https://public-api.viewster.com/series/%s/episodes' % entry_id,
+                video_id, 'Downloading series JSON')
+            entries = [
+                self.url_result(
+                    'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
+                for episode in episodes]
+            title = info.get('Title') or info['Synopsis']['Title']
+            description = info.get('Synopsis', {}).get('Detailed')
+            return self.playlist_result(entries, video_id, title, description)
 
-        entries = []
-        for clip in movie['play_list']:
-            entry = None
-
-            # movielink api
-            link_request = clip.get('link_request')
-            if link_request:
-                request = compat_urllib_request.Request(
-                    'http://api.live.viewster.com/api/v1/movielink?movieid=%(movieid)s&action=%(action)s&paymethod=%(paymethod)s&price=%(price)s&currency=%(currency)s&language=%(language)s&subtitlelanguage=%(subtitlelanguage)s&ischromecast=%(ischromecast)s'
-                    % link_request)
-                request.add_header('Accept', self._ACCEPT_HEADER)
-
-                movie_link = self._download_json(
-                    request, video_id, 'Downloading movie link JSON', fatal=False)
-
-                if movie_link:
-                    formats = self._extract_f4m_formats(
-                        movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
-                    self._sort_formats(formats)
-                    entry = {
-                        'formats': formats,
-                    }
-
-            # direct link
-            clip_url = clip.get('clip_data', {}).get('url')
-            if clip_url:
-                entry = {
-                    'url': clip_url,
-                    'ext': 'mp4',
-                }
-
-            if entry:
-                entry.update({
-                    'id': '%s-%s' % (video_id, clip['canonical_title']),
-                    'title': '%s - %s' % (title, clip['title']),
+        formats = []
+        for media_type in ('application/f4m+xml', 'application/x-mpegURL'):
+            media = self._download_json(
+                'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
+                % (entry_id, compat_urllib_parse.quote(media_type)),
+                video_id, 'Downloading %s JSON' % media_type, fatal=False)
+            if not media:
+                continue
+            video_url = media.get('Uri')
+            if not video_url:
+                continue
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
+                video_url += '&' if '?' in video_url else '?'
+                video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls',
+                    fatal=False  # m3u8 sometimes fail
+                ))
+            else:
+                formats.append({
+                    'url': video_url,
                 })
-                entries.append(entry)
+        self._sort_formats(formats)
 
-        playlist = self.playlist_result(entries, video_id, title, description)
-        playlist['thumbnail'] = thumbnail
-        return playlist
+        synopsis = info.get('Synopsis', {})
+        # Prefer title outside synopsis since it's less messy
+        title = info.get('Title') or synopsis['Title'].strip()
+        description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
+        duration = int_or_none(info.get('Duration'))
+        timestamp = parse_iso8601(info.get('ReleaseDate'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }

From c84683c88bbcb3cfd8d27af54a418035f431371d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 02:08:25 +0600
Subject: [PATCH 0845/2145] [viewster] Strip titles

---
 youtube_dl/extractor/viewster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 03c873608..6ef36290b 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -88,7 +88,7 @@ class ViewsterIE(InfoExtractor):
                 self.url_result(
                     'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
                 for episode in episodes]
-            title = info.get('Title') or info['Synopsis']['Title']
+            title = (info.get('Title') or info['Synopsis']['Title']).strip()
             description = info.get('Synopsis', {}).get('Detailed')
             return self.playlist_result(entries, video_id, title, description)
 
@@ -122,7 +122,7 @@ class ViewsterIE(InfoExtractor):
 
         synopsis = info.get('Synopsis', {})
         # Prefer title outside synopsis since it's less messy
-        title = info.get('Title') or synopsis['Title'].strip()
+        title = (info.get('Title') or synopsis['Title']).strip()
         description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
         duration = int_or_none(info.get('Duration'))
         timestamp = parse_iso8601(info.get('ReleaseDate'))

From f57f84f606b246db4f102fc5bc55e64e4f7a3d60 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:38:40 -0500
Subject: [PATCH 0846/2145] Twitter: get and describe video from status urls

---
 youtube_dl/extractor/twitter.py | 44 +++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1aaa06305..a65252cc6 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -70,3 +70,47 @@ class TwitterCardIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+
+class TwitterIE(TwitterCardIE):
+    _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
+
+    _TESTS = [{
+        'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
+        'md5': '8bbccb487bd7a31349b775915fcd412f',
+        'info_dict': {
+            'id': '614301758345490432',
+            'ext': 'mp4',
+            'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 29.5,
+            'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
+            'uploader': 'banksy',
+            'uploader_id': 'thereaIbanksy',
+        },
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
+        name = username
+        url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
+        webpage = self._download_webpage(url, 'tweet: ' + url)
+        description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
+        title = description.replace('\n', ' ')
+        splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
+        if splitdesc:
+            name, title = splitdesc.groups()
+        title = re.sub(r'\s*https?://[^ ]+', '', title)  # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
+        card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
+        card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'TwitterCard',
+            'uploader_id': username,
+            'uploader': name,
+            'url': card_url,
+            'webpage_url': url,
+            'description': description,
+            'title': username + ' - ' + title,
+        }

From c3dea3f878133f3cbdad9e548609d3077572af66 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:45:36 -0500
Subject: [PATCH 0847/2145] Twittercard: support vmapurl method

---
 youtube_dl/extractor/twitter.py | 47 ++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index a65252cc6..1dd43ff3c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -12,17 +12,30 @@ from ..utils import (
 
 class TwitterCardIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-        'md5': 'a74f50b310c83170319ba16de6955192',
-        'info_dict': {
-            'id': '560070183650213889',
-            'ext': 'mp4',
-            'title': 'TwitterCard',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 30.033,
+    _TESTS = [
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
+            'md5': 'a74f50b310c83170319ba16de6955192',
+            'info_dict': {
+                'id': '560070183650213889',
+                'ext': 'mp4',
+                'title': 'TwitterCard',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 30.033,
+            }
         },
-    }
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
+            'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
+            'info_dict': {
+                'id': '623160978427936768',
+                'ext': 'mp4',
+                'title': 'TwitterCard',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 80.155,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -44,6 +57,20 @@ class TwitterCardIE(InfoExtractor):
                 unescapeHTML(self._search_regex(
                     r'data-player-config="([^"]+)"', webpage, 'data player config')),
                 video_id)
+            if 'playlist' not in config:
+                if 'vmapUrl' in config:
+                    webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
+                    video_url = self._search_regex(
+                        r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
+                    f = {
+                        'url': video_url,
+                    }
+                    ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
+                    if ext:
+                        f['ext'] = ext.group(1)
+                    formats.append(f)
+                    break   # same video regardless of UA
+                continue
 
             video_url = config['playlist'][0]['source']
 

From 9e7e0dffd5e3e3c959e8d99a5e236b9099886fe9 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:56:35 -0500
Subject: [PATCH 0848/2145] Actually add the extractor

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 50da08830..5c03bf8e8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -651,7 +651,7 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
-from .twitter import TwitterCardIE
+from .twitter import TwitterCardIE, TwitterIE
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,

From 70c857b7283270146864f92d159445e038f40781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 11:49:54 +0200
Subject: [PATCH 0849/2145] Credit Zach Bruggeman for the appleconnect
 extractor (#6190)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4fd65f46f..373e05c9f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -133,3 +133,4 @@ Remita Amine
 Aurélio A. Heckert
 Bernhard Minks
 sceext
+Zach Bruggeman

From 981b9cdc8c12d817eaf3ec6b030538c252efe48e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 22:09:01 +0800
Subject: [PATCH 0850/2145] [lecture2go] Improve some regular expressions

---
 youtube_dl/extractor/lecture2go.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 9cf28e31c..fd115ff54 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class Lecture2GoIE(InfoExtractor):
-    _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
         'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
@@ -21,9 +21,9 @@ class Lecture2GoIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<em class="title">(.*?)</em>', webpage, 'title')
+        title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
         video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
-        creator = self._html_search_regex(r'<div id="description">(.*)</div>', webpage, 'creator')
+        creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
 
         return {
             'id': video_id,

From 795704f0f1f963d3f61a7e20074ce41eeb3cdf95 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 22:39:46 +0800
Subject: [PATCH 0851/2145] [lecture2go] Support more formats

---
 youtube_dl/extractor/lecture2go.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index fd115ff54..0075b8a2e 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class Lecture2GoIE(InfoExtractor):
@@ -22,12 +25,26 @@ class Lecture2GoIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
-        video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
+
+        formats = []
+        for url in set(re.findall(r'"src","([^"]+)"', webpage)):
+            ext = determine_ext(url)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(url, video_id))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(url, video_id))
+            else:
+                formats.append({
+                    'url': url,
+                })
+
+        self._sort_formats(formats)
+
         creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'creator': creator
         }

From 1e124295644e3760cd457bb1a6ae717e1cb2c0fc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:05:14 +0800
Subject: [PATCH 0852/2145] [lecture2go] Update _TEST

---
 youtube_dl/extractor/lecture2go.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 0075b8a2e..d0e9416f5 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -11,12 +11,12 @@ class Lecture2GoIE(InfoExtractor):
     _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
-        'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
+        'md5': 'ac02b570883020d208d405d5a3fd2f7f',
         'info_dict': {
             'id': '17473',
-            'ext': 'mp4',
-            'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4',
-            'title': '2 - Endliche Automaten und reguläre Sprachen'
+            'ext': 'flv',
+            'title': '2 - Endliche Automaten und reguläre Sprachen',
+            'creator': 'Frank Heitmann',
         }
     }
 

From 9c29bc69f7d6365835f495dff10f3c5f49671a55 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:15:22 +0800
Subject: [PATCH 0853/2145] [utils] Improve parse_duration

Now dots are parsed. For example '87 Min.'
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e13e11b59..65692a9fb 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -324,6 +324,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('02:03:04'), 7384)
         self.assertEqual(parse_duration('01:02:03:04'), 93784)
         self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
+        self.assertEqual(parse_duration('87 Min.'), 5220)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 942f76d24..ae813099d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1309,10 +1309,10 @@ def parse_duration(s):
     m = re.match(
         r'''(?ix)(?:P?T)?
         (?:
-            (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
+            (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
-            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*|
+            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
             (?:
                 (?:
                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?

From e9c6deffee26db40992293b3055df31804ca7e12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:22:19 +0800
Subject: [PATCH 0854/2145] [lecture2go] Add more metadata fields

---
 youtube_dl/extractor/lecture2go.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index d0e9416f5..a2f9d5c54 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    parse_duration,
+    int_or_none,
+)
 
 
 class Lecture2GoIE(InfoExtractor):
@@ -17,6 +21,7 @@ class Lecture2GoIE(InfoExtractor):
             'ext': 'flv',
             'title': '2 - Endliche Automaten und reguläre Sprachen',
             'creator': 'Frank Heitmann',
+            'duration': 5220,
         }
     }
 
@@ -41,10 +46,16 @@ class Lecture2GoIE(InfoExtractor):
         self._sort_formats(formats)
 
         creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
+        duration = parse_duration(self._html_search_regex(
+            r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'Views:\s*</em>\s*<em[^>]+>(\d+)</em>', webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'creator': creator
+            'creator': creator,
+            'duration': duration,
+            'view_count': view_count,
         }

From 40101dc311909523852a88ba69df76be9b6bc920 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:25:32 +0800
Subject: [PATCH 0855/2145] [lecture2go] Make optional fields non-fatal

---
 youtube_dl/extractor/lecture2go.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index a2f9d5c54..40a3d2346 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -45,7 +45,8 @@ class Lecture2GoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
+        creator = self._html_search_regex(
+            r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator', fatal=False)
         duration = parse_duration(self._html_search_regex(
             r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(

From d609edf4f14e56e3dbb3ffc0be057585c0533666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 22:49:00 +0600
Subject: [PATCH 0856/2145] [udemy] Handle already-logged-in scenario (Closes
 #6327)

---
 youtube_dl/extractor/udemy.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e2bab52fe..4a0eaf65f 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -77,7 +77,11 @@ class UdemyIE(InfoExtractor):
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')
 
-        if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
+        def is_logged(webpage):
+            return any(p in webpage for p in ['href="https://www.udemy.com/user/logout/', '>Logout<'])
+
+        # already logged in
+        if is_logged(login_popup):
             return
 
         login_form = self._form_hidden_inputs('login-form', login_popup)
@@ -95,8 +99,7 @@ class UdemyIE(InfoExtractor):
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if all(logout_pattern not in response
-               for logout_pattern in ['href="https://www.udemy.com/user/logout/', '>Logout<']):
+        if not is_logged(response):
             error = self._html_search_regex(
                 r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                 response, 'error message', default=None)

From ce1bafdce9f7f9e19c172b319a5ae75d1c85759b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 23:49:08 +0600
Subject: [PATCH 0857/2145] [pbs] Clean up title construction rationale

---
 youtube_dl/extractor/pbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ccbe2a9f3..ba6527b80 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -247,8 +247,8 @@ class PBSIE(InfoExtractor):
                 'url': closed_captions_url,
             }]
 
-        # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
-        # 'Full Episode', 'Episode 5', etc. prepend program->title
+        # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc)
+        # Try turning it to 'program - title' naming scheme if possible
         alt_title = info.get('program', {}).get('title')
         if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])

From c7620992d20973320fddb85f936f5dc602d1d708 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 23:49:55 +0600
Subject: [PATCH 0858/2145] [pbs] No need to escape colon

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ba6527b80..a53479aad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -251,7 +251,7 @@ class PBSIE(InfoExtractor):
         # Try turning it to 'program - title' naming scheme if possible
         alt_title = info.get('program', {}).get('title')
         if alt_title:
-            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
+            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title'])
 
         return {
             'id': video_id,

From f79ebf09a24f9bf0e91d0dc26782eeea02f10ad1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 01:56:55 +0800
Subject: [PATCH 0859/2145] Credit @nichdu for Lecture2Go extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 373e05c9f..e75e9885d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -134,3 +134,4 @@ Aurélio A. Heckert
 Bernhard Minks
 sceext
 Zach Bruggeman
+Tjark Saul

From 948199deac4d9ec6ba6bc4359ad43db7951a1c53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 00:30:48 +0600
Subject: [PATCH 0860/2145] [tagesschau] Relax _VALID_URL and simplify

---
 youtube_dl/extractor/tagesschau.py | 62 ++++++++----------------------
 1 file changed, 17 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 636607db5..b84892364 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
@@ -29,57 +29,31 @@ class TagesschauIE(InfoExtractor):
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
-        }
+        },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
-        'md5': '90757268b49ef56deae90c7b48928d58',
-        'info_dict': {
-            'id': '3771',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
-        'md5': '6e3ebdc75e8d67da966a8d06721eda71',
-        'info_dict': {
-            'id': '3827',
-            'ext': 'mp4',
-            'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66',
-            'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
-        'md5': '8a8875a568f0a5ae5ceef93c501a225f',
-        'info_dict': {
-            'id': '3475',
-            'ext': 'mp4',
-            'description': 'md5:ed149f5649cda3dac86813a9d777e131',
-            'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html',
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
-        'md5': 'be4d6f0421f2acd8abe25ea29f6f015b',
-        'info_dict': {
-            'id': '959',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
-        'md5': '42e3757018d9908581481a80cc1806da',
-        'info_dict': {
-            'id': '3299',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
+        'only_matching': True,
     }]
 
     _FORMATS = {
@@ -152,11 +126,9 @@ class TagesschauIE(InfoExtractor):
             thumbnail_fn = self._search_regex(
                 r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
                 webpage, 'thumbnail', fatal=False)
-            # there are some videos without description
-            description = ""
             description = self._html_search_regex(
                 r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', fatal=False, default=None)
+                webpage, 'description', default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From a47b602b0877dcde1b795bf53bfe3629c6595870 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 01:01:04 +0600
Subject: [PATCH 0861/2145] [tagesschau] Add support for audio

---
 youtube_dl/extractor/tagesschau.py | 37 +++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index b84892364..7fd0ba987 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -30,6 +30,16 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
+        'md5': 'aef45de271c4bf0a5db834aa40bf774c',
+        'info_dict': {
+            'id': '18407',
+            'ext': 'mp3',
+            'title': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'description': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'thumbnail': 're:^https?:.*\.jpg$',
+        },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
         'only_matching': True,
@@ -51,9 +61,6 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
         'only_matching': True,
-    }, {
-        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
-        'only_matching': True,
     }]
 
     _FORMATS = {
@@ -73,19 +80,26 @@ class TagesschauIE(InfoExtractor):
             playerpage = self._download_webpage(
                 player_url, display_id, 'Downloading player page')
 
-            medias = re.findall(
-                r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
-                playerpage)
             formats = []
-            for url, ext, res in medias:
+            for media in re.finditer(
+                    r'''(?x)
+                        (?P<q_url>["\'])(?P<url>http://media.+?)(?P=q_url)
+                        ,\s*type:(?P<q_type>["\'])(?P<type>video|audio)/(?P<ext>.+?)(?P=q_type)
+                        (?:,\s*quality:(?P<q_quality>["\'])(?P<quality>.+?)(?P=q_quality))?
+                    ''', playerpage):
+                url = media.group('url')
+                type_ = media.group('type')
+                ext = media.group('ext')
+                res = media.group('quality')
                 f = {
-                    'format_id': res + '_' + ext,
+                    'format_id': '%s_%s' % (res, ext) if res else ext,
                     'url': url,
                     'ext': ext,
+                    'vcodec': 'none' if type_ == 'audio' else None,
                 }
                 f.update(self._FORMATS.get(res, {}))
                 formats.append(f)
-            thumbnail_fn = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+            thumbnail = self._og_search_thumbnail(playerpage)
             title = self._og_search_title(webpage).strip()
             description = self._og_search_description(webpage).strip()
         else:
@@ -123,9 +137,7 @@ class TagesschauIE(InfoExtractor):
                         'filesize_approx': parse_filesize(m.group('filesize_approx')),
                     })
                 formats.append(format)
-            thumbnail_fn = self._search_regex(
-                r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
-                webpage, 'thumbnail', fatal=False)
+            thumbnail = self._og_search_thumbnail(webpage)
             description = self._html_search_regex(
                 r'(?s)<p class="teasertext">(.*?)</p>',
                 webpage, 'description', default=None)
@@ -133,7 +145,6 @@ class TagesschauIE(InfoExtractor):
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
         self._sort_formats(formats)
-        thumbnail = 'http://www.tagesschau.de' + thumbnail_fn
 
         return {
             'id': display_id,

From 3e214851a49992b010ae90ef6dbaed11f70ceb3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 01:01:35 +0600
Subject: [PATCH 0862/2145] [tagesschau] Improve tests

---
 youtube_dl/extractor/tagesschau.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 7fd0ba987..1a7ea98d2 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -18,7 +18,7 @@ class TagesschauIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
             'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
@@ -28,7 +28,7 @@ class TagesschauIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',

From c59b61c0da8fc0fb73d745de59122e290a33b122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 02:13:51 +0600
Subject: [PATCH 0863/2145] [viki] Fix height (Closes #6333)

---
 youtube_dl/extractor/viki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 51cdc6b65..3a7e9a0f2 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -242,8 +242,8 @@ class VikiIE(VikiBaseIE):
 
         formats = []
         for format_id, stream_dict in streams.items():
-            height = self._search_regex(
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
                     formats = self._extract_m3u8_formats(

From 61be92e26a16e24a702c9c65b46f15c375726345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 03:04:23 +0600
Subject: [PATCH 0864/2145] [prosiebensat1] Recognize DRM protected videos
 (#6334)

---
 youtube_dl/extractor/prosiebensat1.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index fec008ce7..0739234c6 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    ExtractorError,
     determine_ext,
     int_or_none,
     unified_strdate,
@@ -224,10 +225,13 @@ class ProSiebenSat1IE(InfoExtractor):
             'ids': clip_id,
         })
 
-        videos = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')
+        video = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')[0]
 
-        duration = float(videos[0]['duration'])
-        source_ids = [source['id'] for source in videos[0]['sources']]
+        if video.get('is_protected') is True:
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        duration = float(video['duration'])
+        source_ids = [source['id'] for source in video['sources']]
         source_ids_str = ','.join(map(str, source_ids))
 
         g = '01!8d8F_)r9]4s[qeuXfP%'

From 993df6bc228ca540bba45e0f5b6041ac9440d695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 03:05:16 +0600
Subject: [PATCH 0865/2145] [prosiebensat1] Modernize

---
 youtube_dl/extractor/prosiebensat1.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 0739234c6..effcf1db3 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     determine_ext,
+    float_or_none,
     int_or_none,
     unified_strdate,
 )
@@ -230,7 +231,7 @@ class ProSiebenSat1IE(InfoExtractor):
         if video.get('is_protected') is True:
             raise ExtractorError('This video is DRM protected.', expected=True)
 
-        duration = float(video['duration'])
+        duration = float_or_none(video.get('duration'))
         source_ids = [source['id'] for source in video['sources']]
         source_ids_str = ','.join(map(str, source_ids))
 

From ca4456eda8f49ed2f939c7ae5cfe55b45777217c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 23:23:38 +0200
Subject: [PATCH 0866/2145] [tagesschau] Remove unused import

---
 youtube_dl/extractor/tagesschau.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 1a7ea98d2..73e7657d4 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize, ExtractorError
+from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):

From 87dc451108b278eb5e5600a0159bdbd337272392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:35:26 +0200
Subject: [PATCH 0867/2145] [youtube] Don't use the DASH manifest from
 'get_video_info' if 'use_cipher_signature' is True (#5118)

Currently they give a 403 Forbidden error.
---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d8b31f98..323681960 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -967,7 +967,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         video_id, note=False,
                         errnote='unable to download video info webpage')
                     get_video_info = compat_parse_qs(video_info_webpage)
-                    add_dash_mpd(get_video_info)
+                    if get_video_info.get('use_cipher_signature') != ['True']:
+                        add_dash_mpd(get_video_info)
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:

From b37317d8b0391651b396fc2dc4e35b59e52fddf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 23:33:49 +0200
Subject: [PATCH 0868/2145] [generic] Unescape HTML escape sequences in
 redirect urls (fixes #6311)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cd133a10c..6d2efb22e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1702,7 +1702,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = compat_urlparse.urljoin(url, found.group(1))
+                new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
                 self.report_following_redirect(new_url)
                 return {
                     '_type': 'url',

From 41597d9bed9eaa5e55d5bb572f2ec3f5a312d392 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 13:39:19 +0800
Subject: [PATCH 0869/2145] [viki] Fix description extraction (closes #6339)

---
 youtube_dl/extractor/viki.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 3a7e9a0f2..e987badbd 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -173,6 +173,19 @@ class VikiIE(VikiBaseIE):
     }, {
         'url': 'http://www.viki.com/player/44699v',
         'only_matching': True,
+    }, {
+        # non-English description
+        'url': 'http://www.viki.com/videos/158036v-love-in-magic',
+        'md5': '1713ae35df5a521b31f6dc40730e7c9c',
+        'info_dict': {
+            'id': '158036v',
+            'ext': 'mp4',
+            'uploader': 'I Planet Entertainment',
+            'upload_date': '20111122',
+            'timestamp': 1321985454,
+            'description': 'md5:44b1e46619df3a072294645c770cef36',
+            'title': 'Love In Magic',
+        },
     }]
 
     def _real_extract(self, url):
@@ -192,8 +205,12 @@ class VikiIE(VikiBaseIE):
                 container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                 title = '%s - %s' % (container_title, title)
 
-        descriptions = video.get('descriptions')
-        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
+        descriptions = video.get('descriptions', {})
+        description = descriptions.get('en')
+        if description is None:
+            filtered_descriptions = list(filter(None, [descriptions.get(k) for k in titles.keys()]))
+            if filtered_descriptions:
+                description = filtered_descriptions[0]
 
         duration = int_or_none(video.get('duration'))
         timestamp = parse_iso8601(video.get('created_at'))

From b73b14f72c8240466b5bded9fb891697549b89ec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 14:02:19 +0800
Subject: [PATCH 0870/2145] [viki] Rewrite dict selection codes

---
 youtube_dl/extractor/viki.py | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index e987badbd..ddbd395c8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -88,6 +88,14 @@ class VikiBaseIE(InfoExtractor):
         if not self._token:
             self.report_warning('Unable to get session token, login has probably failed')
 
+    @staticmethod
+    def dict_selection(dict_obj, preferred_key):
+        if preferred_key in dict_obj:
+            return dict_obj.get(preferred_key)
+
+        filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
+        return filtered_dict[0] if filtered_dict else None
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
@@ -194,23 +202,14 @@ class VikiIE(VikiBaseIE):
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
-        title = None
-        titles = video.get('titles')
-        if titles:
-            title = titles.get('en') or titles[titles.keys()[0]]
+        title = self.dict_selection(video.get('titles', {}), 'en')
         if not title:
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
-            container_titles = video.get('container', {}).get('titles')
-            if container_titles:
-                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
-                title = '%s - %s' % (container_title, title)
+            container_titles = video.get('container', {}).get('titles', {})
+            container_title = self.dict_selection(container_titles, 'en')
+            title = '%s - %s' % (container_title, title)
 
-        descriptions = video.get('descriptions', {})
-        description = descriptions.get('en')
-        if description is None:
-            filtered_descriptions = list(filter(None, [descriptions.get(k) for k in titles.keys()]))
-            if filtered_descriptions:
-                description = filtered_descriptions[0]
+        description = self.dict_selection(video.get('descriptions', {}), 'en')
 
         duration = int_or_none(video.get('duration'))
         timestamp = parse_iso8601(video.get('created_at'))
@@ -316,11 +315,9 @@ class VikiChannelIE(VikiBaseIE):
             'containers/%s.json' % channel_id, channel_id,
             'Downloading channel JSON')
 
-        titles = channel['titles']
-        title = titles.get('en') or titles[titles.keys()[0]]
+        title = self.dict_selection(channel['titles'], 'en')
 
-        descriptions = channel['descriptions']
-        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
+        description = self.dict_selection(channel['descriptions'], 'en')
 
         entries = []
         for video_type in ('episodes', 'clips', 'movies'):

From 59db9f80187c4dc3768f8656ede281347a805e1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 12:09:30 +0200
Subject: [PATCH 0871/2145] [downloader/dash] Improve 'combine_url' (fixes
 #6341)

In some videos the base_url already ends with '/'.
---
 youtube_dl/downloader/dash.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index a4685d307..8b6fa2753 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -37,7 +37,7 @@ class DashSegmentsFD(FileDownloader):
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
                 return target_url
-            return '%s/%s' % (base_url, target_url)
+            return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         with open(tmpfilename, 'wb') as outf:
             append_url_to_file(

From 53b8247cb5b5ac4a7822c82b94ec2f1221a40625 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Thu, 23 Jul 2015 01:38:55 -0500
Subject: [PATCH 0872/2145] NationalGeographic._VALID_URL: work site-wide

Closes #6343.
---
 youtube_dl/extractor/nationalgeographic.py | 35 +++++++++++++++-------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index f793b72f5..6fc9e7b05 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -8,18 +8,30 @@ from ..utils import (
 
 
 class NationalGeographicIE(InfoExtractor):
-    _VALID_URL = r'http://video\.nationalgeographic\.com/video/.*?'
+    _VALID_URL = r'http://video\.nationalgeographic\.com/.*?'
 
-    _TEST = {
-        'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
-        'info_dict': {
-            'id': '4DmDACA6Qtk_',
-            'ext': 'flv',
-            'title': 'Mating Crabs Busted by Sharks',
-            'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+    _TESTS = [
+        {
+            'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+            'info_dict': {
+                'id': '4DmDACA6Qtk_',
+                'ext': 'flv',
+                'title': 'Mating Crabs Busted by Sharks',
+                'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+            },
+            'add_ie': ['ThePlatform'],
         },
-        'add_ie': ['ThePlatform'],
-    }
+        {
+            'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
+            'info_dict': {
+                'id': '_JeBD_D7PlS5',
+                'ext': 'flv',
+                'title': 'The Real Jaws',
+                'description': 'md5:8d3e09d9d53a85cd397b4b21b2c77be6',
+            },
+            'add_ie': ['ThePlatform'],
+        },
+    ]
 
     def _real_extract(self, url):
         name = url_basename(url)
@@ -37,5 +49,6 @@ class NationalGeographicIE(InfoExtractor):
 
         return self.url_result(smuggle_url(
             'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
-            # For some reason, the normal links don't work and we must force the use of f4m
+            # For some reason, the normal links don't work and we must force
+            # the use of f4m
             {'force_smil_url': True}))

From 297a564beeb20ca8b00d94f5707532110631f409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 13:20:21 +0200
Subject: [PATCH 0873/2145] [youtube] Extract end_time

---
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e8751877..1272834c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -185,6 +185,8 @@ class InfoExtractor(object):
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the url.
+    end_time:       Time in seconds where the reproduction should end, as
+                    specified in the url.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index afbd34f4a..117ef2e77 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -319,7 +319,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -332,6 +332,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
+                'end_time': 9,
             }
         },
         {
@@ -893,12 +894,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else 'https')
 
         start_time = None
+        end_time = None
         parsed_url = compat_urllib_parse_urlparse(url)
         for component in [parsed_url.fragment, parsed_url.query]:
             query = compat_parse_qs(component)
-            if 't' in query:
+            if start_time is None and 't' in query:
                 start_time = parse_duration(query['t'][0])
-                break
+            if end_time is None and 'end' in query:
+                end_time = parse_duration(query['end'][0])
 
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
@@ -1267,6 +1270,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'formats': formats,
             'is_live': is_live,
             'start_time': start_time,
+            'end_time': end_time,
         }
 
 

From 2929fa0e79dfd3a1366e7e23eb4344bc93dd3a10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 13:21:18 +0200
Subject: [PATCH 0874/2145] [youtube] Also look into the 'start' field for
 start_time

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 117ef2e77..462d244d8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -900,6 +900,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             query = compat_parse_qs(component)
             if start_time is None and 't' in query:
                 start_time = parse_duration(query['t'][0])
+            if start_time is None and 'start' in query:
+                start_time = parse_duration(query['start'][0])
             if end_time is None and 'end' in query:
                 end_time = parse_duration(query['end'][0])
 

From d3f007af183a0f5ed278602128e6bba3cc1350b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 22:04:47 +0600
Subject: [PATCH 0875/2145] [daylimotion] Adapt to player v5 and modernize
 (Closes #6151, closes #6250)

---
 youtube_dl/extractor/dailymotion.py | 160 ++++++++++++++++++++--------
 1 file changed, 117 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 1a41c0db1..8fcae7402 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -13,8 +13,10 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    determine_ext,
     int_or_none,
     orderedSet,
+    parse_iso8601,
     str_to_int,
     unescapeHTML,
 )
@@ -28,10 +30,12 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
+    def _download_webpage_no_ff(self, url, *args, **kwargs):
+        request = self._build_request(url)
+        return self._download_webpage(request, *args, **kwargs)
+
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    """Information Extractor for Dailymotion"""
-
     _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
@@ -50,10 +54,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'info_dict': {
                 'id': 'x2iuewm',
                 'ext': 'mp4',
-                'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
-                'upload_date': '20150306',
+                'description': 'Several come bundled with the Steam Controller.',
+                'thumbnail': 're:^https?:.*\.(?:jpg|png)$',
                 'duration': 74,
+                'timestamp': 1425657362,
+                'upload_date': '20150306',
+                'uploader': 'IGN',
+                'uploader_id': 'xijv66',
+                'age_limit': 0,
+                'view_count': int,
+                'comment_count': int,
             }
         },
         # Vevo video
@@ -87,38 +98,106 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.dailymotion.com/video/%s' % video_id
 
-        # Retrieve video webpage to extract further information
-        request = self._build_request(url)
-        webpage = self._download_webpage(request, video_id)
-
-        # Extract URL, uploader and title from webpage
-        self.report_extraction(video_id)
-
-        # It may just embed a vevo video:
-        m_vevo = re.search(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
-            webpage)
-        if m_vevo is not None:
-            vevo_id = m_vevo.group('id')
-            self.to_screen('Vevo video detected: %s' % vevo_id)
-            return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+        webpage = self._download_webpage_no_ff(
+            'https://www.dailymotion.com/video/%s' % video_id, video_id)
 
         age_limit = self._rta_search(webpage)
 
-        video_upload_date = None
-        mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
-        if mobj is not None:
-            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
+
+        view_count = str_to_int(self._search_regex(
+            [r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:(\d+)"',
+             r'video_views_count[^>]+>\s+([\d\.,]+)'],
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
+            webpage, 'comment count', fatal=False))
+
+        player_v5 = self._search_regex(
+            r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+            webpage, 'player v5', default=None)
+        if player_v5:
+            player = self._parse_json(player_v5, video_id)
+            metadata = player['metadata']
+            formats = []
+            for quality, media_list in metadata['qualities'].items():
+                for media in media_list:
+                    media_url = media.get('url')
+                    if not media_url:
+                        continue
+                    type_ = media.get('type')
+                    if type_ == 'application/vnd.lumberjack.manifest':
+                        continue
+                    if type_ == 'application/x-mpegURL' or determine_ext(media_url) == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls'))
+                    else:
+                        f = {
+                            'url': media_url,
+                            'format_id': quality,
+                        }
+                        m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
+                        if m:
+                            f.update({
+                                'width': int(m.group('width')),
+                                'height': int(m.group('height')),
+                            })
+                        formats.append(f)
+            self._sort_formats(formats)
+
+            title = metadata['title']
+            duration = int_or_none(metadata.get('duration'))
+            timestamp = int_or_none(metadata.get('created_time'))
+            thumbnail = metadata.get('poster_url')
+            uploader = metadata.get('owner', {}).get('screenname')
+            uploader_id = metadata.get('owner', {}).get('id')
+
+            subtitles = {}
+            for subtitle_lang, subtitle in metadata.get('subtitles', {}).get('data', {}).items():
+                subtitles[subtitle_lang] = [{
+                    'ext': determine_ext(subtitle_url),
+                    'url': subtitle_url,
+                } for subtitle_url in subtitle.get('urls', [])]
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'timestamp': timestamp,
+                'uploader': uploader,
+                'uploader_id': uploader_id,
+                'age_limit': age_limit,
+                'view_count': view_count,
+                'comment_count': comment_count,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # vevo embed
+        vevo_id = self._search_regex(
+            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
+            webpage, 'vevo embed', default=None)
+        if vevo_id:
+            return self.url_result('vevo:%s' % vevo_id, 'Vevo')
+
+        # fallback old player
+        embed_page = self._download_webpage_no_ff(
+            'https://www.dailymotion.com/embed/video/%s' % video_id,
+            video_id, 'Downloading embed page')
+
+        timestamp = parse_iso8601(self._html_search_meta(
+            'video:release_date', webpage, 'upload date'))
+
+        info = self._parse_json(
+            self._search_regex(
+                r'var info = ({.*?}),$', embed_page,
+                'video info', flags=re.MULTILINE),
+            video_id)
 
-        embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
-        embed_request = self._build_request(embed_url)
-        embed_page = self._download_webpage(
-            embed_request, video_id, 'Downloading embed page')
-        info = self._search_regex(r'var info = ({.*?}),$', embed_page,
-                                  'video info', flags=re.MULTILINE)
-        info = json.loads(info)
         if info.get('error') is not None:
             msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
             raise ExtractorError(msg, expected=True)
@@ -139,16 +218,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     'width': width,
                     'height': height,
                 })
-        if not formats:
-            raise ExtractorError('Unable to extract video URL')
+        self._sort_formats(formats)
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, webpage)
 
-        view_count = str_to_int(self._search_regex(
-            r'video_views_count[^>]+>\s+([\d\.,]+)',
-            webpage, 'view count', fatal=False))
-
         title = self._og_search_title(webpage, default=None)
         if title is None:
             title = self._html_search_regex(
@@ -159,8 +233,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'id': video_id,
             'formats': formats,
             'uploader': info['owner.screenname'],
-            'upload_date': video_upload_date,
+            'timestamp': timestamp,
             'title': title,
+            'description': description,
             'subtitles': video_subtitles,
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
@@ -201,9 +276,9 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     def _extract_entries(self, id):
         video_ids = []
         for pagenum in itertools.count(1):
-            request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
-            webpage = self._download_webpage(request,
-                                             id, 'Downloading page %s' % pagenum)
+            webpage = self._download_webpage_no_ff(
+                self._PAGE_TEMPLATE % (id, pagenum),
+                id, 'Downloading page %s' % pagenum)
 
             video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
 
@@ -286,8 +361,7 @@ class DailymotionCloudIE(DailymotionBaseInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = self._build_request(url)
-        webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage_no_ff(url, video_id)
 
         title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
 

From ba911137fa53737b663cb2879d357bc8a35ab558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 22:50:59 +0600
Subject: [PATCH 0876/2145] [rts] Add support for articles with videos on rhs
 (Closes #6332)

---
 youtube_dl/extractor/rts.py | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 9fbe239d8..12639f08b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -19,7 +19,16 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        rts:(?P<rts_id>\d+)|
+                        https?://
+                            (?:www\.)?rts\.ch/
+                            (?:
+                                (?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|
+                                play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+)
+                            )
+                    )'''
 
     _TESTS = [
         {
@@ -122,6 +131,15 @@ class RTSIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        {
+            # article with videos on rhs
+            'url': 'http://www.rts.ch/sport/hockey/6693917-hockey-davos-decroche-son-31e-titre-de-champion-de-suisse.html',
+            'info_dict': {
+                'id': '6693917',
+                'title': 'Hockey: Davos décroche son 31e titre de champion de Suisse',
+            },
+            'playlist_mincount': 5,
+        },
         {
             'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
             'only_matching': True,
@@ -130,7 +148,7 @@ class RTSIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('id') or m.group('id_new')
+        video_id = m.group('rts_id') or m.group('id') or m.group('id_new')
         display_id = m.group('display_id') or m.group('display_id_new')
 
         def download_json(internal_id):
@@ -143,6 +161,15 @@ class RTSIE(InfoExtractor):
         # video_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
             page = self._download_webpage(url, display_id)
+
+            # article with videos on rhs
+            videos = re.findall(
+                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:rts:video:(\d+)"',
+                page)
+            if videos:
+                entries = [self.url_result('rts:%s' % video_urn, 'RTS') for video_urn in videos]
+                return self.playlist_result(entries, video_id, self._og_search_title(page))
+
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')

From a8b7b26068fa8de9983ffef995ea6cd4fd3ce90a Mon Sep 17 00:00:00 2001
From: corone17 <reino@degeelebosch.nl>
Date: Wed, 22 Jul 2015 22:38:04 +0200
Subject: [PATCH 0877/2145] Undo adaptive -> flash workaround

For a couple of days now rtlXL's non-DRM adaptive (m3u8) streams don't work anymore. By undoing the adaptive -> flash workaround youtube-dl returns the DRM streams again. Too bad for foreigners (geoblock).
The progressive streams still work fine.
---
 youtube_dl/extractor/rtlnl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e0c530d64..518f7a7b4 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -86,12 +86,12 @@ class RtlNlIE(InfoExtractor):
         # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
         # this adaptive -> flash workaround is not required in general, but it also
         # allows bypassing georestriction therefore is retained for now.
-        videopath = material['videopath'].replace('/adaptive/', '/flash/')
+        videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
-        video_urlpart = videopath.split('/flash/')[1][:-5]
+        video_urlpart = videopath.split('/adaptive/')[1][:-5]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([

From a9e8f60ef686eeeb3d31f2e5211ce176dc8f5298 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 23:20:24 +0600
Subject: [PATCH 0878/2145] [rtlnl] Update unencrypted streams comment (#6337)

---
 youtube_dl/extractor/rtlnl.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 518f7a7b4..543d94417 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -82,10 +82,15 @@ class RtlNlIE(InfoExtractor):
 
         meta = info.get('meta', {})
 
-        # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
-        # this adaptive -> flash workaround is not required in general, but it also
-        # allows bypassing georestriction therefore is retained for now.
+        # m3u8 streams are encrypted and may not be handled properly by older ffmpeg/avconv.
+        # To workaround this previously adaptive -> flash trick was used to obtain
+        # unencrypted m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
+        # and bypass georestrictions as well.
+        # Currently, unencrypted m3u8 playlists are (intentionally?) invalid and therefore
+        # unusable albeit can be fixed by simple string replacement (see
+        # https://github.com/rg3/youtube-dl/pull/6337)
+        # Since recent ffmpeg and avconv handle encrypted streams just fine encrypted
+        # streams are used now.
         videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 

From 10952eb2cf2c86c841e46690a3bfa6a39c82d67d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 23:37:45 +0600
Subject: [PATCH 0879/2145] [extractor/common] Consistent URL spelling

---
 youtube_dl/extractor/common.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1272834c5..14b9b4fe2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -65,7 +65,7 @@ class InfoExtractor(object):
 
                     Potential fields:
                     * url        Mandatory. The URL of the video file
-                    * ext        Will be calculated from url if missing
+                    * ext        Will be calculated from URL if missing
                     * format     A human-readable description of the format
                                  ("mp4 container with h264/opus").
                                  Calculated from the format_id, width, height.
@@ -155,7 +155,7 @@ class InfoExtractor(object):
                     lower to higher preference, each element is a dictionary
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
-                        * "url": A url pointing to the subtitles file
+                        * "url": A URL pointing to the subtitles file
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
     duration:       Length of the video in seconds, as an integer.
@@ -176,7 +176,7 @@ class InfoExtractor(object):
                                      Set to "root" to indicate that this is a
                                      comment to the original video.
     age_limit:      Age restriction for the video, as an integer (years)
-    webpage_url:    The url to the video webpage, if given to youtube-dl it
+    webpage_url:    The URL to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
@@ -184,9 +184,9 @@ class InfoExtractor(object):
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
-                    specified in the url.
+                    specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
-                    specified in the url.
+                    specified in the URL.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -505,7 +505,7 @@ class InfoExtractor(object):
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):
-        """Returns a url that points to a page that should be processed"""
+        """Returns a URL that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
                       'url': url,
@@ -639,7 +639,7 @@ class InfoExtractor(object):
         return unescapeHTML(escaped)
 
     def _og_search_thumbnail(self, html, **kargs):
-        return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
+        return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
 
     def _og_search_description(self, html, **kargs):
         return self._og_search_property('description', html, fatal=False, **kargs)
@@ -1120,7 +1120,7 @@ class InfoExtractor(object):
 class SearchInfoExtractor(InfoExtractor):
     """
     Base class for paged search queries extractors.
-    They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
+    They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
     Instances should define _SEARCH_KEY and _MAX_RESULTS.
     """
 

From 660f9459dabfc01c7235b4cc57f9c561de522e07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 00:17:36 +0600
Subject: [PATCH 0880/2145] [canalplus] Fix m3u8 videos extension

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 699b4f7d0..8671236dd 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -106,7 +106,7 @@ class CanalplusIE(InfoExtractor):
                 continue
             format_id = fmt.tag
             if format_id == 'HLS':
-                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
+                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4')
                 for fmt in hls_formats:
                     fmt['preference'] = preference(format_id)
                 formats.extend(hls_formats)

From f3f0b8e4030a07e07385afe6ebb4485ac33e5357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 00:21:13 +0600
Subject: [PATCH 0881/2145] [canalplus] Modernize

---
 youtube_dl/extractor/canalplus.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 8671236dd..57e0cda2c 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -106,15 +106,11 @@ class CanalplusIE(InfoExtractor):
                 continue
             format_id = fmt.tag
             if format_id == 'HLS':
-                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4')
-                for fmt in hls_formats:
-                    fmt['preference'] = preference(format_id)
-                formats.extend(hls_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', preference=preference(format_id)))
             elif format_id == 'HDS':
-                hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
-                for fmt in hds_formats:
-                    fmt['preference'] = preference(format_id)
-                formats.extend(hds_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url + '?hdcore=2.11.3', video_id, preference=preference(format_id)))
             else:
                 formats.append({
                     'url': format_url,

From d97f5cd795b0bd8b6bd636b899bdbc8b10d5907b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 02:56:54 +0600
Subject: [PATCH 0882/2145] [bbccouk] Make more robust (Closes #6345)

---
 youtube_dl/extractor/bbccouk.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 5825d2867..b2e5f7418 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -210,12 +210,12 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_video(self, media, programme_id):
         formats = []
-        vbr = int(media.get('bitrate'))
+        vbr = int_or_none(media.get('bitrate'))
         vcodec = media.get('encoding')
         service = media.get('service')
-        width = int(media.get('width'))
-        height = int(media.get('height'))
-        file_size = int(media.get('media_file_size'))
+        width = int_or_none(media.get('width'))
+        height = int_or_none(media.get('height'))
+        file_size = int_or_none(media.get('media_file_size'))
         for connection in self._extract_connections(media):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
@@ -232,7 +232,7 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_audio(self, media, programme_id):
         formats = []
-        abr = int(media.get('bitrate'))
+        abr = int_or_none(media.get('bitrate'))
         acodec = media.get('encoding')
         service = media.get('service')
         for connection in self._extract_connections(media):
@@ -300,7 +300,7 @@ class BBCCoUkIE(InfoExtractor):
                     if kind != 'programme' and kind != 'radioProgramme':
                         continue
                     programme_id = item.get('vpid')
-                    duration = int(item.get('duration'))
+                    duration = int_or_none(item.get('duration'))
                     formats, subtitles = self._download_media_selector(programme_id)
                 return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
@@ -332,7 +332,7 @@ class BBCCoUkIE(InfoExtractor):
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
             programme_id = item.get('identifier')
-            duration = int(item.get('duration'))
+            duration = int_or_none(item.get('duration'))
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles

From 2b2ee140c3c6f08b4078cc6a5a289e5e74bec2b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 03:40:24 +0600
Subject: [PATCH 0883/2145] [dailymotion:user] Fix _VALID_URL (Closes #6346)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 8fcae7402..5a4987772 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -302,7 +302,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1a117a77287e7dbd4d92f29062dabcf4efb86cb5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 24 Jul 2015 12:00:20 +0100
Subject: [PATCH 0884/2145] [clipfish] extract mp4 video link

---
 youtube_dl/extractor/clipfish.py | 37 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index a5c3cb7c6..09dfaac60 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-import time
-import xml.etree.ElementTree
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_duration,
+    int_or_none,
+    js_to_json,
+    determine_ext,
 )
 
 
@@ -17,37 +15,40 @@ class ClipfishIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
     _TEST = {
         'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
-        'md5': '2521cd644e862936cf2e698206e47385',
+        'md5': '79bc922f3e8a9097b3d68a93780fd475',
         'info_dict': {
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
             'duration': 82,
-        },
-        'skip': 'Blocked in the US'
+        }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_info = self._parse_json(
+            js_to_json(self._html_search_regex('var videoObject = ({[^}]+?})', webpage, 'videoObject')),
+            video_id
+        )
+        info_url = self._parse_json(
+            js_to_json(self._html_search_regex('var globalFlashvars = ({[^}]+?})', webpage, 'globalFlashvars')),
+            video_id
+        )['data']
 
-        info_url = ('http://www.clipfish.de/devxml/videoinfo/%s?ts=%d' %
-                    (video_id, int(time.time())))
         doc = self._download_xml(
             info_url, video_id, note='Downloading info page')
         title = doc.find('title').text
         video_url = doc.find('filename').text
-        if video_url is None:
-            xml_bytes = xml.etree.ElementTree.tostring(doc)
-            raise ExtractorError('Cannot find video URL in document %r' %
-                                 xml_bytes)
         thumbnail = doc.find('imageurl').text
-        duration = parse_duration(doc.find('duration').text)
+        duration = int_or_none(video_info['length'])
+        formats = [{'url': video_info['videourl']},{'url': video_url}]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
         }

From 12434026574bcaaaa705c31ef14428cc91a5efad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Jul 2015 21:29:44 +0800
Subject: [PATCH 0885/2145] [dailymotion:playlist] Detect problematic
 redirection (fixes #6347)

---
 youtube_dl/extractor/dailymotion.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 5a4987772..85d945509 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -30,6 +30,10 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
+    def _download_webpage_handle_no_ff(self, url, *args, **kwargs):
+        request = self._build_request(url)
+        return self._download_webpage_handle(request, *args, **kwargs)
+
     def _download_webpage_no_ff(self, url, *args, **kwargs):
         request = self._build_request(url)
         return self._download_webpage(request, *args, **kwargs)
@@ -275,10 +279,17 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
     def _extract_entries(self, id):
         video_ids = []
+        processed_urls = set()
         for pagenum in itertools.count(1):
-            webpage = self._download_webpage_no_ff(
-                self._PAGE_TEMPLATE % (id, pagenum),
-                id, 'Downloading page %s' % pagenum)
+            page_url = self._PAGE_TEMPLATE % (id, pagenum)
+            webpage, urlh = self._download_webpage_handle_no_ff(
+                page_url, id, 'Downloading page %s' % pagenum)
+            if urlh.geturl() in processed_urls:
+                self.report_warning('Stopped at duplicated page %s, which is the same as %s' % (
+                    page_url, urlh.geturl()), id)
+                break
+
+            processed_urls.add(urlh.geturl())
 
             video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
 
@@ -311,6 +322,17 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': 'Rémi Gaillard',
         },
         'playlist_mincount': 100,
+    }, {
+        'url': 'http://www.dailymotion.com/user/UnderProject',
+        'info_dict': {
+            'id': 'UnderProject',
+            'title': 'UnderProject',
+        },
+        'playlist_mincount': 1800,
+        'expected_warnings': [
+            'Stopped at duplicated page',
+        ],
+        'skip': 'Takes too long time',
     }]
 
     def _real_extract(self, url):

From 593ddd851b87eb7ac327a9216800162f07b2acca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 24 Jul 2015 14:46:45 +0100
Subject: [PATCH 0886/2145] [letv] fix height

---
 youtube_dl/extractor/letv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ba2ae8085..a28abb0f0 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -15,6 +15,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
+    int_or_none,
 )
 
 
@@ -134,7 +135,7 @@ class LetvIE(InfoExtractor):
                 }
 
                 if format_id[-1:] == 'p':
-                    url_info_dict['height'] = format_id[:-1]
+                    url_info_dict['height'] = int_or_none(format_id[:-1])
 
                 urls.append(url_info_dict)
 

From 678e436f2e77f1ae3a57c4b5d1fc3d74342ab412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 02:09:34 +0600
Subject: [PATCH 0887/2145] [youtube] Handle empty allowed regions (Closes
 #6351)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 229fc3a0f..4023a6e50 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -993,7 +993,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if 'reason' in video_info:
                 if 'The uploader has not made this video available in your country.' in video_info['reason']:
                     regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
-                    if regions_allowed is not None:
+                    if regions_allowed:
                         raise ExtractorError('YouTube said: This video is available in %s only' % (
                             ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
                             expected=True)

From b14fa8e6874818a3f210b2a67cf53345000defdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 15:47:53 +0600
Subject: [PATCH 0888/2145] [soundcloud:set] Defer download link resolve
 (Closes #6354)

---
 youtube_dl/extractor/soundcloud.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 118ca4832..cdee8e2a3 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -282,9 +282,11 @@ class SoundcloudSetIE(SoundcloudIE):
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]
+
         return {
             '_type': 'playlist',
-            'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
+            'entries': entries,
             'id': '%s' % info['id'],
             'title': info['title'],
         }

From 40a2d17052e9b542eb3c360a0ce067d244e07fb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 15:48:44 +0600
Subject: [PATCH 0889/2145] [soundcloud:playlist] Defer download link resolve

---
 youtube_dl/extractor/soundcloud.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index cdee8e2a3..0a6c9fe72 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -381,9 +381,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
-        entries = [
-            self._extract_info_dict(t, quiet=True, secret_token=token)
-            for t in data['tracks']]
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]
 
         return {
             '_type': 'playlist',

From eab7faa0c1e8511bc91c64347d0dffc28c94f101 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:39:01 +0800
Subject: [PATCH 0890/2145] [ir90tv] Test (?:www\.)? part in _VALID_URL

---
 youtube_dl/extractor/ir90tv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index b79529b1b..880a6e32f 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class Ir90TvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
         'md5': '411dbd94891381960cb9e13daa47a869',
         'info_dict': {
@@ -15,7 +15,10 @@ class Ir90TvIE(InfoExtractor):
             'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 9700cd9097445d598515dc41fb3cb9421403b9b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:42:40 +0800
Subject: [PATCH 0891/2145] [ir90tv] Improve title extraction

---
 youtube_dl/extractor/ir90tv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 880a6e32f..92333c3ea 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import remove_start
 
 
 class Ir90TvIE(InfoExtractor):
@@ -24,8 +25,8 @@ class Ir90TvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
+        title = remove_start(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), '90tv.ir :: ')
 
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')

From 7523647391969f8d747ba0fc178592f7f3d5e453 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:43:07 +0800
Subject: [PATCH 0892/2145] [ir90tv] PEP8

---
 youtube_dl/extractor/ir90tv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 92333c3ea..6418d6178 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -37,6 +37,6 @@ class Ir90TvIE(InfoExtractor):
             'url': video_url,
             'id': video_id,
             'title': title,
-            'video_url' : video_url,
-            'thumbnail' : thumbnail,
+            'video_url': video_url,
+            'thumbnail': thumbnail,
         }

From 2c7c721933e53ece49bee0140d2dad9a8219d6e4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:48:00 +0800
Subject: [PATCH 0893/2145] [ir90tv] Optional fields should be non-fatal

---
 youtube_dl/extractor/ir90tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 6418d6178..214bcd5b5 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -31,7 +31,7 @@ class Ir90TvIE(InfoExtractor):
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
-        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)
 
         return {
             'url': video_url,

From 9afa1770d1a6835bc8fee48dc86cd1a702d1f67a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 20:21:42 +0600
Subject: [PATCH 0894/2145] [bbc] Improve playlist extraction, refactor, expand
 support and document

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/bbc.py      | 375 +++++++++++++++++++++----------
 2 files changed, 259 insertions(+), 121 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bc61cbdc5..d77ed3ba2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -43,7 +43,10 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbc import BBCCoUkIE, BBCNewsIE
+from .bbc import (
+    BBCCoUkIE,
+    BBCIE,
+)
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 86327d8ed..2a0901ee4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1,15 +1,18 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_duration,
+    float_or_none,
     int_or_none,
+    parse_duration,
+    parse_iso8601,
 )
 from ..compat import compat_HTTPError
-import re
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -17,7 +20,7 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
 
     _TESTS = [
         {
@@ -264,16 +267,21 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
+        return self._download_media_selector_url(
+            self._MEDIASELECTOR_URL % programme_id, programme_id)
+
+    def _download_media_selector_url(self, url, programme_id=None):
         try:
             media_selection = self._download_xml(
-                self.mediaselector_url % programme_id,
-                programme_id, 'Downloading media selection XML')
+                url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                 media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
+        return self._process_media_selector(media_selection, programme_id)
 
+    def _process_media_selector(self, media_selection, programme_id):
         formats = []
         subtitles = None
 
@@ -312,10 +320,21 @@ class BBCCoUkIE(InfoExtractor):
                 raise
 
         # fallback to legacy playlist
-        playlist = self._download_xml(
-            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
-            playlist_id, 'Downloading legacy playlist XML')
+        return self._process_legacy_playlist(playlist_id)
 
+    def _process_legacy_playlist_url(self, url, display_id):
+        playlist = self._download_legacy_playlist_url(url, display_id)
+        return self._extract_from_legacy_playlist(playlist, display_id)
+
+    def _process_legacy_playlist(self, playlist_id):
+        return self._process_legacy_playlist_url(
+            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)
+
+    def _download_legacy_playlist_url(self, url, playlist_id=None):
+        return self._download_xml(
+            url, playlist_id, 'Downloading legacy playlist XML')
+
+    def _extract_from_legacy_playlist(self, playlist, playlist_id):
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:
             reason = no_items.get('reason')
@@ -335,8 +354,23 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
-            programme_id = item.get('identifier')
+
+            def get_programme_id(item):
+                def get_from_attributes(item):
+                    for p in('identifier', 'group'):
+                        value = item.get(p)
+                        if value and re.match(r'^[pb][\da-z]{7}$', value):
+                            return value
+                get_from_attributes(item)
+                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                if mediator is not None:
+                    return get_from_attributes(mediator)
+
+            programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
+            # TODO: programme_id can be None and media items can be incorporated right inside
+            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+            # as f4m and m3u8
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles
@@ -383,175 +417,276 @@ class BBCCoUkIE(InfoExtractor):
         }
 
 
-class BBCNewsIE(BBCCoUkIE):
-    IE_NAME = 'bbc.com'
-    IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.com/.+?/(?P<id>[^/]+)$'
+class BBCIE(BBCCoUkIE):
+    IE_NAME = 'bbc'
+    IE_DESC = 'BBC'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # fails with notukerror for some videos
+    #_MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{
+        # article with multiple videos embedded with data-media-meta containing
+        # playlist.sxml, externalId and no direct video links
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
             'id': 'world-europe-32668511',
             'title': 'Russia stages massive WW2 parade despite Western boycott',
+            'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
         },
         'playlist_count': 2,
     }, {
+        # article with multiple videos embedded with data-media-meta (more videos)
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
             'title': 'Farnborough Airshow: Video highlights',
+            'description': 'BBC reports and video highlights at the Farnborough Airshow.',
         },
         'playlist_count': 9,
+        'skip': 'Save time',
     }, {
+        # single video embedded with mediaAssetPage.init()
         'url': 'http://www.bbc.com/news/world-europe-32041533',
-        'note': 'Video',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
+            'timestamp': 1427219242,
             'upload_date': '20150324',
-            'uploader': 'BBC News',
         },
         'params': {
+            # rtmp download
             'skip_download': True,
         }
     }, {
+        # article with single video embedded with data-media-meta containing
+        # direct video links (for now these are extracted) and playlist.xml (with
+        # media items as f4m and m3u8 - currently unsupported)
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
-        'note': 'Video',
         'info_dict': {
-            'id': 'NA',
+            'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
-            'title': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
-            'description': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
+            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
             'duration': 47,
+            'timestamp': 1434397334,
             'upload_date': '20150615',
-            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
         }
     }, {
+        # single video embedded with mediaAssetPage.init() (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
-        'note': 'Video',
         'info_dict': {
-            'id': '39275083',
+            'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
-            'title': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
-            'description': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
+            'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
             'duration': 87,
+            'timestamp': 1434713142,
             'upload_date': '20150619',
-            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video story with digitalData
+        'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
+        'info_dict': {
+            'id': 'p02q6gc4',
+            'ext': 'flv',
+            'title': 'Sri Lanka’s spicy secret',
+            'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
+            'timestamp': 1437674293,
+            'upload_date': '20150723',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video story without digitalData
+        'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
+        'info_dict': {
+            'id': 'p018zqqg',
+            'ext': 'flv',
+            'title': 'Hyundai Santa Fe Sport: Rock star',
+            'description': 'md5:b042a26142c4154a6e472933cf20793d',
+            'timestamp': 1368473503,
+            'upload_date': '20130513',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist.sxml URL
+        'url': 'http://www.bbc.com/sport/0/football/33653409',
+        'info_dict': {
+            'id': 'p02xycnp',
+            'ext': 'flv',
+            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
+            'duration': 140,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist URL from weather section
+        'url': 'http://www.bbc.com/weather/features/33601775',
+        'only_matching': True,
+    }, {
+        # custom redirection to www.bbc.com
+        'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+
+    def _extract_from_media_meta(self, media_meta, video_id):
+        # Direct links to media in media metadata (e.g.
+        # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+        # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
+        source_files = media_meta.get('sourceFiles')
+        if source_files:
+            return [{
+                'url': f['url'],
+                'format_id': format_id,
+                'ext': f.get('encoding'),
+                'tbr': float_or_none(f.get('bitrate'), 1000),
+                'filesize': int_or_none(f.get('filesize')),
+            } for format_id, f in source_files.items() if f.get('url')], []
+
+        programme_id = media_meta.get('externalId')
+        if programme_id:
+            return self._download_media_selector(programme_id)
+
+        # Process playlist.sxml as legacy playlist
+        href = media_meta.get('href')
+        if href:
+            playlist = self._download_legacy_playlist_url(href)
+            _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
+            return formats, subtitles
+
+        return [], []
+
     def _real_extract(self, url):
-        list_id = self._match_id(url)
-        webpage = self._download_webpage(url, list_id)
+        playlist_id = self._match_id(url)
 
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'list title')
+        webpage = self._download_webpage(url, playlist_id)
 
-        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
-        if pubdate:
-            pubdate = pubdate.replace('-', '')
-
-        ret = []
-        jsent = []
-
-        # works with bbc.com/news/something-something-123456 articles
-        jsent = map(
-            lambda m: self._parse_json(m, list_id),
-            re.findall(r"data-media-meta='({[^']+})'", webpage)
-        )
-
-        if len(jsent) == 0:
-            # http://www.bbc.com/news/video_and_audio/international
-            # and single-video articles
-            masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
-            if masset:
-                jmasset = self._parse_json(masset, list_id)
-                for key, val in jmasset.get('videos', {}).items():
-                    for skey, sval in val.items():
-                        sval['id'] = skey
-                        jsent.append(sval)
-
-        if len(jsent) == 0:
-            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-            # in http://www.bbc.com/news/video_and_audio/international
-            # prone to breaking if entries have sourceFiles list
-            jsent = map(
-                lambda m: self._parse_json(m, list_id),
-                re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-            )
-
-        if len(jsent) == 0:
-            raise ExtractorError('No video found', expected=True)
-
-        for jent in jsent:
-            programme_id = jent.get('externalId')
-            xml_url = jent.get('href')
-
-            title = jent.get('caption', '')
-            if title == '':
-                title = list_title
-
-            duration = parse_duration(jent.get('duration'))
-            description = list_title
-            if jent.get('caption', '') != '':
-                description += ' - ' + jent.get('caption')
-            thumbnail = None
-            if jent.get('image') is not None:
-                thumbnail = jent['image'].get('href')
-
-            formats = []
-            subtitles = []
-
-            if programme_id:
-                formats, subtitles = self._download_media_selector(programme_id)
-            elif jent.get('sourceFiles') is not None:
-                # mediaselector not used at
-                # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
-                for key, val in jent['sourceFiles'].items():
-                    formats.append({
-                        'ext': val.get('encoding'),
-                        'url': val.get('url'),
-                        'filesize': int(val.get('filesize')),
-                        'format_id': key
-                    })
-            elif xml_url:
-                # Cheap fallback
-                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-                formats, subtitles = self._download_media_selector(programme_id)
-
-            if len(formats) == 0:
-                raise ExtractorError('unsupported json media entry.\n    ' + str(jent) + '\n')
+        timestamp = parse_iso8601(self._search_regex(
+            [r'"datePublished":\s*"([^"]+)',
+             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
+            webpage, 'date', default=None))
 
+        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
+        playlist = self._search_regex(
+            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
+            webpage, 'playlist', default=None)
+        if playlist:
+            programme_id, title, description, duration, formats, subtitles = \
+                self._process_legacy_playlist_url(playlist, playlist_id)
             self._sort_formats(formats)
-
-            id = jent.get('id') if programme_id is None else programme_id
-            if id is None:
-                id = 'NA'
-
-            ret.append({
-                'id': id,
-                'uploader': 'BBC News',
-                'upload_date': pubdate,
+            return {
+                'id': programme_id,
                 'title': title,
                 'description': description,
-                'thumbnail': thumbnail,
                 'duration': duration,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        programme_id = self._search_regex(
+            [r'data-video-player-vpid="([\da-z]{8})"',
+             r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+            webpage, 'vpid', default=None)
+        if programme_id:
+            formats, subtitles = self._download_media_selector(programme_id)
+            self._sort_formats(formats)
+            # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
+            digital_data = self._parse_json(
+                self._search_regex(
+                    r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
+                programme_id, fatal=False)
+            page_info = digital_data.get('page', {}).get('pageInfo', {})
+            title = page_info.get('pageName') or self._og_search_title(webpage)
+            description = page_info.get('description') or self._og_search_description(webpage)
+            timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
+            return {
+                'id': programme_id,
+                'title': title,
+                'description': description,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        playlist_title = self._html_search_regex(
+            r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
+        playlist_description = self._og_search_description(webpage)
+
+        # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
+        medias = list(filter(None, map(
+            lambda s: self._parse_json(s, playlist_id, fatal=False),
+            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+
+        if not medias:
+            # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
+            media_asset_page = self._parse_json(
+                self._search_regex(
+                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                playlist_id)
+            medias = []
+            for video in media_asset_page.get('videos', {}).values():
+                medias.extend(video.values())
+
+        entries = []
+        for num, media_meta in enumerate(medias, start=1):
+            formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
+            if not formats:
+                continue
+            self._sort_formats(formats)
+
+            video_id = media_meta.get('externalId')
+            if not video_id:
+                video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)
+
+            title = media_meta.get('caption')
+            if not title:
+                title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)
+
+            duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))
+
+            images = []
+            for image in media_meta.get('images', {}).values():
+                images.extend(image.values())
+            if 'image' in media_meta:
+                images.append(media_meta['image'])
+
+            thumbnails = [{
+                'url': image.get('href'),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            } for image in images]
+
+            entries.append({
+                'id': video_id,
+                'title': title,
+                'thumbnails': thumbnails,
+                'duration': duration,
+                'timestamp': timestamp,
                 'formats': formats,
                 'subtitles': subtitles,
             })
 
-        if len(ret) > 0:
-            return self.playlist_result(ret, list_id, list_title)
-        raise ExtractorError('No video found', expected=True)
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

From cf7e015f250d806916bed1ed2a1dfd8d943c05c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 21:30:34 +0600
Subject: [PATCH 0895/2145] [youtube] Add support for multifeed videos

---
 youtube_dl/extractor/youtube.py | 126 +++++++++++++++++++++++++-------
 1 file changed, 99 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4023a6e50..afe0a781b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,9 +33,11 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    smuggle_url,
     str_to_int,
     unescapeHTML,
     unified_strdate,
+    unsmuggle_url,
     uppercase_escape,
     ISO3166Utils,
 )
@@ -558,6 +560,59 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'format': '135',  # bestvideo
             }
         },
+        {
+            # Multifeed videos (multiple cameras), URL is for Main Camera
+            'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
+            'info_dict': {
+                'id': 'jqWvoWXjCVs',
+                'title': 'teamPGP: Rocket League Noob Stream',
+                'description': 'md5:dc7872fb300e143831327f1bae3af010',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': 'jqWvoWXjCVs',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': '6h8e8xoXJzg',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': 'PUOgX5z9xZw',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': 'teuwxikvS5k',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (zim)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }],
+            'params': {
+                'skip_download': True,
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -889,6 +944,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return formats
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         proto = (
             'http' if self._downloader.params.get('prefer_insecure', False)
             else 'https')
@@ -1005,6 +1062,48 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     '"token" parameter not in video info for unknown reason',
                     video_id=video_id)
 
+        # title
+        if 'title' in video_info:
+            video_title = video_info['title'][0]
+        else:
+            self._downloader.report_warning('Unable to extract video title')
+            video_title = '_'
+
+        # description
+        video_description = get_element_by_id("eow-description", video_webpage)
+        if video_description:
+            video_description = re.sub(r'''(?x)
+                <a\s+
+                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    title="([^"]+)"\s+
+                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    class="yt-uix-redirect-link"\s*>
+                [^<]+
+                </a>
+            ''', r'\1', video_description)
+            video_description = clean_html(video_description)
+        else:
+            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
+            if fd_mobj:
+                video_description = unescapeHTML(fd_mobj.group(1))
+            else:
+                video_description = ''
+
+        if not smuggled_data.get('force_singlefeed', False) and 'multifeed_metadata_list' in video_info:
+            entries = []
+            multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+            for feed in multifeed_metadata_list.split(','):
+                feed_data = compat_parse_qs(feed)
+                entries.append({
+                    '_type': 'url_transparent',
+                    'ie_key': 'Youtube',
+                    'url': smuggle_url(
+                        '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                        {'force_singlefeed': True}),
+                    'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                })
+            return self.playlist_result(entries, video_id, video_title, video_description)
+
         if 'view_count' in video_info:
             view_count = int(video_info['view_count'][0])
         else:
@@ -1030,13 +1129,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
-        # title
-        if 'title' in video_info:
-            video_title = video_info['title'][0]
-        else:
-            self._downloader.report_warning('Unable to extract video title')
-            video_title = '_'
-
         # thumbnail image
         # We try first to get a high quality image:
         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
@@ -1072,26 +1164,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        # description
-        video_description = get_element_by_id("eow-description", video_webpage)
-        if video_description:
-            video_description = re.sub(r'''(?x)
-                <a\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    title="([^"]+)"\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="yt-uix-redirect-link"\s*>
-                [^<]+
-                </a>
-            ''', r'\1', video_description)
-            video_description = clean_html(video_description)
-        else:
-            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
-            if fd_mobj:
-                video_description = unescapeHTML(fd_mobj.group(1))
-            else:
-                video_description = ''
-
         def _extract_count(count_name):
             return str_to_int(self._search_regex(
                 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'

From d8f0a9ecea6aab5b148d06904934c6a504ba3b03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:17:06 +0600
Subject: [PATCH 0896/2145] [youtube] Respect noplaylist for multifeed videos

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index afe0a781b..bcd27408d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1089,7 +1089,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if not smuggled_data.get('force_singlefeed', False) and 'multifeed_metadata_list' in video_info:
+        if (not self._downloader.params.get('noplaylist') and
+                not smuggled_data.get('force_singlefeed', False) and
+                'multifeed_metadata_list' in video_info):
             entries = []
             multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
             for feed in multifeed_metadata_list.split(','):

From 51da40e6218f1dda2fc61650c308194e9b4acbc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:19:54 +0600
Subject: [PATCH 0897/2145] [bbc] PEP8

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2a0901ee4..4b23f82ca 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -423,7 +423,7 @@ class BBCIE(BBCCoUkIE):
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
     # fails with notukerror for some videos
-    #_MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
     _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{

From 7a896817226405a772baa3808d63062d4ad11c94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:32:54 +0600
Subject: [PATCH 0898/2145] [bbc] Skip DASH until supported

---
 youtube_dl/extractor/bbc.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4b23f82ca..66e52641b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -172,6 +172,7 @@ class BBCCoUkIE(InfoExtractor):
         supplier = connection.get('supplier')
         if protocol == 'http':
             href = connection.get('href')
+            transfer_format = connection.get('transferFormat')
             # ASX playlist
             if supplier == 'asx':
                 for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
@@ -179,6 +180,9 @@ class BBCCoUkIE(InfoExtractor):
                         'url': ref,
                         'format_id': 'ref%s_%s' % (i, supplier),
                     })
+            # Skip DASH until supported
+            elif transfer_format == 'dash':
+                pass
             # Direct link
             else:
                 formats.append({

From a62fd1af275a125b77d782e821724d6c6c896b69 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 25 Jul 2015 18:13:44 +0100
Subject: [PATCH 0899/2145] [shahid] improve info extraction

---
 youtube_dl/extractor/shahid.py | 42 +++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 04de866f4..da3141f11 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,7 +1,8 @@
 from .common import InfoExtractor
 from ..utils import (
-    get_element_by_id,
+    js_to_json,
     ExtractorError,
+    int_or_none
 )
 
 class ShahidIE(InfoExtractor):
@@ -12,8 +13,9 @@ class ShahidIE(InfoExtractor):
             'info_dict': {
                 'id': '108084',
                 'ext': 'm3u8',
-                'title': 'بسم الله',
-                'description': 'بسم الله'
+                'title': 'خواطر الموسم 11 الحلقة 1',
+                'description': 'بسم الله',
+                'duration': 1166,
             },
             'params': {
                 # m3u8 download
@@ -30,16 +32,33 @@ class ShahidIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        json_data = self._parse_json(
-            get_element_by_id('jsonld', webpage),
+        player_info = ''
+        for line in self._search_regex( 'var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
+            if '+' not in line and '(' not in line and ')' not in line:
+                player_info += line
+        player_info = self._parse_json(js_to_json(player_info), video_id)
+        video_id = player_info['id']
+        player_type = player_info['playerType']
+
+        video_info = self._download_json(
+            player_info['url'] + '/' + player_type + '/' + video_id + 
+                '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
             video_id
-        )
-        title = json_data['name']
-        thumbnail = json_data.get('image')
-        categories = json_data.get('genre')
-        description = json_data.get('description')
+        )['data']
+        if video_info['error']:
+            for error in video_info['error']:
+                raise ExtractorError(error)
+        video_info = video_info[player_type]
+        if video_info.get('availabilities').get('plus'):
+            raise ExtractorError('plus members only')
+        title = video_info['title']
+        thumbnail = video_info.get('thumbnailUrl')
+        categories = [category['name'] for category in video_info.get('genres')]
+        description = video_info.get('description')
+        duration = int_or_none(video_info.get('duration'))
+
         player_json_data = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-'+video_id+'.type-player.html',
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + player_info['type'] + '.html',
             video_id
         )['data']
         if 'url' in player_json_data:
@@ -55,5 +74,6 @@ class ShahidIE(InfoExtractor):
             'thumbnail': thumbnail,
             'categories': categories,
             'description': description,
+            'duration': duration,
             'formats': formats,
         }

From dfaba1ab9586a7c47017bd897ffda0b795a1d27e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 25 Jul 2015 19:14:18 +0100
Subject: [PATCH 0900/2145] [shahid] fix variable name

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index da3141f11..b3b45da24 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -64,7 +64,7 @@ class ShahidIE(InfoExtractor):
         if 'url' in player_json_data:
             m3u8_url = player_json_data['url']
         else:
-            for error in json_data['error'].values():
+            for error in player_json_data['error'].values():
                 raise ExtractorError(error)
             return
         formats = self._extract_m3u8_formats(m3u8_url, video_id)

From 5bdec59de15b9bde73a3077a6b9ce517c10b9906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jul 2015 09:51:54 +0600
Subject: [PATCH 0901/2145] [comcarcoff] Add support for singleshots (Closes
 #6366)

---
 youtube_dl/extractor/comcarcoff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 9c25b2223..81f3d7697 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -36,7 +36,7 @@ class ComCarCoffIE(InfoExtractor):
             webpage, 'full data json'))
 
         video_id = full_data['activeVideo']['video']
-        video_data = full_data['videos'][video_id]
+        video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {

From aeb7b41d44313b6bb007b2f1cd0fc8cda84e59d5 Mon Sep 17 00:00:00 2001
From: tippfeler <tippfeler@gmx.de>
Date: Sun, 26 Jul 2015 12:57:06 +0200
Subject: [PATCH 0902/2145] [spiegel] Accept iframe urls

Closes #6370.
---
 youtube_dl/extractor/spiegel.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index b868241d5..5bd3c0087 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -9,7 +9,7 @@ from .spiegeltv import SpiegeltvIE
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed|-iframe)?(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -39,6 +39,9 @@ class SpiegelIE(InfoExtractor):
             'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
             'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
         }
+    }, {
+        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4c6bd5b5b61adfd912e14f8d704fde47628d164e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 26 Jul 2015 14:14:28 +0200
Subject: [PATCH 0903/2145] [youtube] Use 'vp8' and 'vp9' in lowercase (fixes
 #6358)

That's how YouTube reports them in their DASH manifest.
---
 youtube_dl/extractor/youtube.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4023a6e50..0e411bfb6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -281,13 +281,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'VP9'},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
@@ -297,11 +297,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From 2c7ed247967f3563d26879f2206e0f54dfdf9b23 Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Sun, 26 Jul 2015 16:37:51 +0200
Subject: [PATCH 0904/2145] Remove redundant (and wrong) class parameters

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ae813099d..88f9f9070 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -576,11 +576,9 @@ class ContentTooShortError(Exception):
     download is too small for what the server announced first, indicating
     the connection was probably interrupted.
     """
-    # Both in bytes
-    downloaded = None
-    expected = None
 
     def __init__(self, downloaded, expected):
+        # Both in bytes
         self.downloaded = downloaded
         self.expected = expected
 

From cb23bcba294563857561914a19e7d06990c71829 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 27 Jul 2015 04:24:04 -0500
Subject: [PATCH 0905/2145] BBCIE: Fix missing .mp4 formats on news sites

Prior to merge BBCNewsIE from pr #6026 was rewritten into BBCIE in 9afa177. Support was added for non-news
sites (/travel, /sports), however support for the news mediaselector was removed to support these sites.
This removed support for .mp4.

    pr #6026 ( news site, news mediaseletor : http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ )
    format code                                          extension  resolution note
    journalism_nonuk_stream_h264_flv_200_akamai          flv        400x224    h264@ 176k, 1001.77KiB
    journalism_nonuk_stream_h264_flv_400_akamai          flv        400x224    h264@ 512k, 2.85MiB
    journalism_nonuk_stream_h264_flv_med_akamai          flv        640x360    h264@ 800k, 4.43MiB
    journalism_nonuk_stream_h264_flv_hi_akamai           flv        688x384    h264@1500k, 8.34MiB
    journalism_world_stream_h264_http_200_sis_news_http  mp4        400x224    h264@ 176k, 1001.77KiB
    journalism_world_stream_h264_http_400_sis_news_http  mp4        400x224    h264@ 512k, 2.85MiB
    journalism_world_stream_h264_http_hi_sis_news_http   mp4        688x384    h264@1500k, 8.34MiB (best)

    # 9afa177 ( same url, non-news mediaselector: http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ )
    format code                                  extension  resolution note
    journalism_nonuk_stream_h264_flv_lo_akamai   flv        384x216    h264@ 496k, 2.76MiB
    journalism_nonuk_stream_h264_flv_med_akamai  flv        640x360    h264@ 800k, 4.43MiB
    journalism_nonuk_stream_h264_flv_hi_akamai   flv        688x384    h264@1500k, 8.34MiB (best)

This change corrects the above, by trying /mediaselector/5 if /mediaselector/4 fails.
---
 youtube_dl/extractor/bbc.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 66e52641b..0f0ea7cfd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -14,7 +14,6 @@ from ..utils import (
 )
 from ..compat import compat_HTTPError
 
-
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
@@ -271,8 +270,16 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
-        return self._download_media_selector_url(
-            self._MEDIASELECTOR_URL % programme_id, programme_id)
+        try:
+            return self._download_media_selector_url(
+                self._MEDIASELECTOR_URL % programme_id, programme_id)
+        except ExtractorError as e:
+            if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror':
+                 # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/
+                 return self._download_media_selector_url(
+                     self._MEDIASELECTOR_ALT_URL % programme_id, programme_id)
+            else:
+                 raise
 
     def _download_media_selector_url(self, url, programme_id=None):
         try:
@@ -297,7 +304,6 @@ class BBCCoUkIE(InfoExtractor):
                 formats.extend(self._extract_video(media, programme_id))
             elif kind == 'captions':
                 subtitles = self.extract_subtitles(media, programme_id)
-
         return formats, subtitles
 
     def _download_playlist(self, playlist_id):
@@ -426,9 +432,10 @@ class BBCIE(BBCCoUkIE):
     IE_DESC = 'BBC'
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    # fails with notukerror for some videos
-    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+    # fails with notukerror for some videos ( non news sites such as bbc.com/travel )
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # limited selection of formats but may work where the above does not
+    _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{
         # article with multiple videos embedded with data-media-meta containing
@@ -455,7 +462,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
             'timestamp': 1427219242,
@@ -515,7 +522,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
             'timestamp': 1368473503,
@@ -530,7 +537,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
             'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
             'duration': 140,

From 88ed52aec9f2e622188f304d74f2f5568b0caa1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jul 2015 22:05:51 +0600
Subject: [PATCH 0906/2145] [bbc] Add support for direct bbc.co.uk embeds

---
 youtube_dl/extractor/bbc.py     | 34 +++++++++++++++++++++++++++++----
 youtube_dl/extractor/generic.py |  8 --------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 66e52641b..c0433eabd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -450,6 +450,14 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 9,
         'skip': 'Save time',
+    }, {
+        # article with multiple videos embedded with `new SMP()`
+        'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
+        'info_dict': {
+            'id': '3662a707-0af9-3149-963f-47bea720b460',
+            'title': 'BBC Blogs - Adam Curtis - BUGGER',
+        },
+        'playlist_count': 18,
     }, {
         # single video embedded with mediaAssetPage.init()
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -637,12 +645,30 @@ class BBCIE(BBCCoUkIE):
 
         playlist_title = self._html_search_regex(
             r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
-        playlist_description = self._og_search_description(webpage)
+        playlist_description = self._og_search_description(webpage, default=None)
+
+        def extract_all(pattern):
+            return list(filter(None, map(
+                lambda s: self._parse_json(s, playlist_id, fatal=False),
+                re.findall(pattern, webpage))))
+
+        # Multiple video article (e.g.
+        # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]*)?'
+        entries = []
+        for match in extract_all(r'new\s+SMP\(({.+?})\)'):
+            embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
+            if embed_url and re.match(EMBED_URL, embed_url):
+                entries.append(embed_url)
+        entries.extend(re.findall(
+            r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
+        if entries:
+            return self.playlist_result(
+                [self.url_result(entry, 'BBCCoUk') for entry in entries],
+                playlist_id, playlist_title, playlist_description)
 
         # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
-        medias = list(filter(None, map(
-            lambda s: self._parse_json(s, playlist_id, fatal=False),
-            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+        medias = extract_all(r"data-media-meta='({[^']+})'")
 
         if not medias:
             # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6d2efb22e..8cef61c3c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -276,14 +276,6 @@ class GenericIE(InfoExtractor):
                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
             },
         },
-        # BBC iPlayer embeds
-        {
-            'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
-            'info_dict': {
-                'title': 'BBC - Blogs -  Adam Curtis - BUGGER',
-            },
-            'playlist_mincount': 18,
-        },
         # RUTV embed
         {
             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',

From d247a2c8bf2fe4e071cb35a2e2777fe16acecab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Jul 2015 02:06:27 +0600
Subject: [PATCH 0907/2145] [bbc] Fix regex

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c0433eabd..01d07c9c0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -654,7 +654,7 @@ class BBCIE(BBCCoUkIE):
 
         # Multiple video article (e.g.
         # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
-        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]*)?'
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
         entries = []
         for match in extract_all(r'new\s+SMP\(({.+?})\)'):
             embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')

From 289bbb350e11ae1460db515560345088d7c58663 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Jul 2015 11:28:33 +0200
Subject: [PATCH 0908/2145] release 2015.07.28

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 73445137f..657935dc6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -51,6 +51,7 @@
  - **bambuser:channel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **BeatportPro**
  - **Beeg**
@@ -224,6 +225,7 @@
  - **InternetVideoArchive**
  - **IPrima**
  - **iqiyi**: 爱奇艺
+ - **Ir90Tv**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -252,6 +254,7 @@
  - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
+ - **Lecture2Go**
  - **Letv**: 乐视网
  - **LetvPlaylist**
  - **LetvTv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 280afdd7f..fa157cadb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.21'
+__version__ = '2015.07.28'

From f171bc8b59ecf4560dd4076be56570a4f090d519 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:14:06 +0300
Subject: [PATCH 0909/2145] [youtube] save keywords in info jason when
 --write-info-json is used

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0e411bfb6..15e327ec8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,6 +1072,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
+        m = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
+        , video_webpage, re.DOTALL | re.IGNORECASE);
+        video_tags = ", ".join(m)
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
@@ -1259,6 +1262,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'title': video_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
+            'tags' : video_tags,
             'categories': video_categories,
             'subtitles': video_subtitles,
             'automatic_captions': automatic_captions,

From a6f774e9015995393a086273df8db1d7b0c098c4 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:29:13 +0300
Subject: [PATCH 0910/2145] [youtube]: tags key in info jason is now a list

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 15e327ec8..c0fafbfd5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,9 +1072,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        m = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
+        video_tags = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
         , video_webpage, re.DOTALL | re.IGNORECASE);
-        video_tags = ", ".join(m)
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:

From 5316bf7487b608b7c085950ff2fb0444f2c36dc0 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:30:42 +0300
Subject: [PATCH 0911/2145] Documented tags as a possible dict key

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 14b9b4fe2..a227aeb9c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -187,6 +187,7 @@ class InfoExtractor(object):
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
+    tags:           A list of keywords attached to the video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 

From 95d8f7ea12f40a986b541ee17cdf384dbedcfa55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:26:16 +0600
Subject: [PATCH 0912/2145] [fragment] Generalize fragmented media file
 downloader

---
 youtube_dl/downloader/fragment.py | 110 ++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 youtube_dl/downloader/fragment.py

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
new file mode 100644
index 000000000..5f9d6796d
--- /dev/null
+++ b/youtube_dl/downloader/fragment.py
@@ -0,0 +1,110 @@
+from __future__ import division, unicode_literals
+
+import os
+import time
+
+from .common import FileDownloader
+from .http import HttpFD
+from ..utils import (
+    encodeFilename,
+    sanitize_open,
+)
+
+
+class HttpQuietDownloader(HttpFD):
+    def to_screen(self, *args, **kargs):
+        pass
+
+
+class FragmentFD(FileDownloader):
+    """
+    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+    """
+
+    def _prepare_and_start_frag_download(self, ctx):
+        self._prepare_frag_download(ctx)
+        self._start_frag_download(ctx)
+
+    def _prepare_frag_download(self, ctx):
+        self.to_screen('[%s] Total fragments: %d' % (self.FD_NAME, ctx['total_frags']))
+        self.report_destination(ctx['filename'])
+        dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': True,
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
+                'test': self.params.get('test', False),
+            }
+        )
+        tmpfilename = self.temp_name(ctx['filename'])
+        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
+        ctx.update({
+            'dl': dl,
+            'dest_stream': dest_stream,
+            'tmpfilename': tmpfilename,
+        })
+
+    def _start_frag_download(self, ctx):
+        total_frags = ctx['total_frags']
+        # This dict stores the download progress, it's updated by the progress
+        # hook
+        state = {
+            'status': 'downloading',
+            'downloaded_bytes': 0,
+            'frag_index': 0,
+            'frag_count': total_frags,
+            'filename': ctx['filename'],
+            'tmpfilename': ctx['tmpfilename'],
+        }
+        start = time.time()
+        ctx['started'] = start
+
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            frag_total_bytes = s.get('total_bytes', 0)
+            if s['status'] == 'finished':
+                state['downloaded_bytes'] += frag_total_bytes
+                state['frag_index'] += 1
+
+            estimated_size = (
+                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['frag_index'] + 1) * total_frags)
+            time_now = time.time()
+            state['total_bytes_estimate'] = estimated_size
+            state['elapsed'] = time_now - start
+
+            if s['status'] == 'finished':
+                progress = self.calc_percent(state['frag_index'], total_frags)
+            else:
+                frag_downloaded_bytes = s['downloaded_bytes']
+                frag_progress = self.calc_percent(frag_downloaded_bytes,
+                                                  frag_total_bytes)
+                progress = self.calc_percent(state['frag_index'], total_frags)
+                progress += frag_progress / float(total_frags)
+
+                state['eta'] = self.calc_eta(
+                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                state['speed'] = s.get('speed')
+            self._hook_progress(state)
+
+        ctx['dl'].add_progress_hook(frag_progress_hook)
+
+        return start
+
+    def _finish_frag_download(self, ctx):
+        ctx['dest_stream'].close()
+        elapsed = time.time() - ctx['started']
+        self.try_rename(ctx['tmpfilename'], ctx['filename'])
+        fsize = os.path.getsize(encodeFilename(ctx['filename']))
+
+        self._hook_progress({
+            'downloaded_bytes': fsize,
+            'total_bytes': fsize,
+            'filename': ctx['filename'],
+            'status': 'finished',
+            'elapsed': elapsed,
+        })

From ab81ef8fa7d0f9e814890024140c7ed9587d3151 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:27:50 +0600
Subject: [PATCH 0913/2145] [f4m] Implement f4m fd in terms of fragment fd

---
 youtube_dl/downloader/f4m.py | 98 +++++++-----------------------------
 1 file changed, 17 insertions(+), 81 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b1a858c45..275564b59 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -7,8 +7,7 @@ import os
 import time
 import xml.etree.ElementTree as etree
 
-from .common import FileDownloader
-from .http import HttpFD
+from .fragment import FragmentFD
 from ..compat import (
     compat_urlparse,
     compat_urllib_error,
@@ -16,8 +15,6 @@ from ..compat import (
 from ..utils import (
     struct_pack,
     struct_unpack,
-    encodeFilename,
-    sanitize_open,
     xpath_text,
 )
 
@@ -226,16 +223,13 @@ def _add_ns(prop):
     return '{http://ns.adobe.com/f4m/1.0}%s' % prop
 
 
-class HttpQuietDownloader(HttpFD):
-    def to_screen(self, *args, **kargs):
-        pass
-
-
-class F4mFD(FileDownloader):
+class F4mFD(FragmentFD):
     """
     A downloader for f4m manifests or AdobeHDS.
     """
 
+    FD_NAME = 'f4m'
+
     def _get_unencrypted_media(self, doc):
         media = doc.findall(_add_ns('media'))
         if not media:
@@ -288,7 +282,7 @@ class F4mFD(FileDownloader):
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
-        self.to_screen('[download] Downloading f4m manifest')
+        self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
         manifest = self.ydl.urlopen(man_url).read()
 
         doc = etree.fromstring(manifest)
@@ -320,67 +314,20 @@ class F4mFD(FileDownloader):
         # For some akamai manifests we'll need to add a query to the fragment url
         akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
-        self.report_destination(filename)
-        http_dl = HttpQuietDownloader(
-            self.ydl,
-            {
-                'continuedl': True,
-                'quiet': True,
-                'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
-                'test': self.params.get('test', False),
-            }
-        )
-        tmpfilename = self.temp_name(filename)
-        (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+        ctx = {
+            'filename': filename,
+            'total_frags': total_frags,
+        }
+
+        self._prepare_frag_download(ctx)
+
+        dest_stream = ctx['dest_stream']
 
         write_flv_header(dest_stream)
         if not live:
             write_metadata_tag(dest_stream, metadata)
 
-        # This dict stores the download progress, it's updated by the progress
-        # hook
-        state = {
-            'status': 'downloading',
-            'downloaded_bytes': 0,
-            'frag_index': 0,
-            'frag_count': total_frags,
-            'filename': filename,
-            'tmpfilename': tmpfilename,
-        }
-        start = time.time()
-
-        def frag_progress_hook(s):
-            if s['status'] not in ('downloading', 'finished'):
-                return
-
-            frag_total_bytes = s.get('total_bytes', 0)
-            if s['status'] == 'finished':
-                state['downloaded_bytes'] += frag_total_bytes
-                state['frag_index'] += 1
-
-            estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes) /
-                (state['frag_index'] + 1) * total_frags)
-            time_now = time.time()
-            state['total_bytes_estimate'] = estimated_size
-            state['elapsed'] = time_now - start
-
-            if s['status'] == 'finished':
-                progress = self.calc_percent(state['frag_index'], total_frags)
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                frag_progress = self.calc_percent(frag_downloaded_bytes,
-                                                  frag_total_bytes)
-                progress = self.calc_percent(state['frag_index'], total_frags)
-                progress += frag_progress / float(total_frags)
-
-                state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
-                state['speed'] = s.get('speed')
-            self._hook_progress(state)
-
-        http_dl.add_progress_hook(frag_progress_hook)
+        self._start_frag_download(ctx)
 
         frags_filenames = []
         while fragments_list:
@@ -391,9 +338,9 @@ class F4mFD(FileDownloader):
                 url += '?' + akamai_pv.strip(';')
             if info_dict.get('extra_param_to_segment_url'):
                 url += info_dict.get('extra_param_to_segment_url')
-            frag_filename = '%s-%s' % (tmpfilename, name)
+            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = http_dl.download(frag_filename, {'url': url})
+                success = ctx['dl'].download(frag_filename, {'url': url})
                 if not success:
                     return False
                 with open(frag_filename, 'rb') as down:
@@ -425,20 +372,9 @@ class F4mFD(FileDownloader):
                     msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
                     self.report_warning(msg)
 
-        dest_stream.close()
+        self._finish_frag_download(ctx)
 
-        elapsed = time.time() - start
-        self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
             os.remove(frag_file)
 
-        fsize = os.path.getsize(encodeFilename(filename))
-        self._hook_progress({
-            'downloaded_bytes': fsize,
-            'total_bytes': fsize,
-            'filename': filename,
-            'status': 'finished',
-            'elapsed': elapsed,
-        })
-
         return True

From f9a5affad968c6629c022f489dc3c1561a4c57de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:28:30 +0600
Subject: [PATCH 0914/2145] [hls] Implement hlsnative fd in terms of fragment
 fd

---
 youtube_dl/downloader/hls.py | 81 +++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8be4f4249..60dca0ab1 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -4,12 +4,11 @@ import os
 import re
 import subprocess
 
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_request,
-)
+from .fragment import FragmentFD
+
+from ..compat import compat_urlparse
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
     encodeArgument,
     encodeFilename,
@@ -51,54 +50,50 @@ class HlsFD(FileDownloader):
             return False
 
 
-class NativeHlsFD(FileDownloader):
+class NativeHlsFD(FragmentFD):
     """ A more limited implementation that does not require ffmpeg """
 
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
+    FD_NAME = 'hlsnative'
 
-        self.to_screen(
-            '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id'])
-        data = self.ydl.urlopen(url).read()
-        s = data.decode('utf-8', 'ignore')
-        segment_urls = []
+    def real_download(self, filename, info_dict):
+        man_url = info_dict['url']
+        self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
+        manifest = self.ydl.urlopen(man_url).read()
+
+        s = manifest.decode('utf-8', 'ignore')
+        fragment_urls = []
         for line in s.splitlines():
             line = line.strip()
             if line and not line.startswith('#'):
                 segment_url = (
                     line
                     if re.match(r'^https?://', line)
-                    else compat_urlparse.urljoin(url, line))
-                segment_urls.append(segment_url)
-
-        is_test = self.params.get('test', False)
-        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
-        byte_counter = 0
-        with open(tmpfilename, 'wb') as outf:
-            for i, segurl in enumerate(segment_urls):
-                self.to_screen(
-                    '[hlsnative] %s: Downloading segment %d / %d' %
-                    (info_dict['id'], i + 1, len(segment_urls)))
-                seg_req = compat_urllib_request.Request(segurl)
-                if remaining_bytes is not None:
-                    seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
-
-                segment = self.ydl.urlopen(seg_req).read()
-                if remaining_bytes is not None:
-                    segment = segment[:remaining_bytes]
-                    remaining_bytes -= len(segment)
-                outf.write(segment)
-                byte_counter += len(segment)
-                if remaining_bytes is not None and remaining_bytes <= 0:
+                    else compat_urlparse.urljoin(man_url, line))
+                fragment_urls.append(segment_url)
+                # We only download the first fragment during the test
+                if self.params.get('test', False):
                     break
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': byte_counter,
+        ctx = {
             'filename': filename,
-            'status': 'finished',
-        })
-        self.try_rename(tmpfilename, filename)
+            'total_frags': len(fragment_urls),
+        }
+
+        self._prepare_and_start_frag_download(ctx)
+
+        frags_filenames = []
+        for i, frag_url in enumerate(fragment_urls):
+            frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+            success = ctx['dl'].download(frag_filename, {'url': frag_url})
+            if not success:
+                return False
+            with open(frag_filename, 'rb') as down:
+                ctx['dest_stream'].write(down.read())
+            frags_filenames.append(frag_filename)
+
+        self._finish_frag_download(ctx)
+
+        for frag_file in frags_filenames:
+            os.remove(frag_file)
+
         return True

From 864f24bd2c0cf9bde034812a2049c3750c1bb05c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 03:43:03 +0600
Subject: [PATCH 0915/2145] [extractor/common] Add _meta_regex and clarify tags
 field

---
 youtube_dl/extractor/common.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a227aeb9c..d54866d1f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -181,13 +181,13 @@ class InfoExtractor(object):
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
                     ["Sports", "Berlin"]
+    tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
-    tags:           A list of keywords attached to the video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -631,6 +631,12 @@ class InfoExtractor(object):
             template % (content_re, property_re),
         ]
 
+    @staticmethod
+    def _meta_regex(prop):
+        return r'''(?isx)<meta
+                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
+
     def _og_search_property(self, prop, html, name=None, **kargs):
         if name is None:
             name = 'OpenGraph %s' % prop
@@ -661,9 +667,7 @@ class InfoExtractor(object):
         if display_name is None:
             display_name = name
         return self._html_search_regex(
-            r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
-                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
+            self._meta_regex(name),
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):

From 000b6b5ae5cc214906effe4ac5b78b579bc7db70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 03:43:32 +0600
Subject: [PATCH 0916/2145] [youtube] Improve tags extraction and add test

---
 youtube_dl/extractor/youtube.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c0fafbfd5..4c449fd74 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -329,6 +329,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20121002',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
+                'tags': ['youtube-dl'],
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
@@ -343,7 +344,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
-                'description': 'md5:fea86fda2d5a5784273df5c7cc994d9f',
+                'description': 'md5:782e8651347686cba06e58f71ab51773',
+                'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
+                         'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
+                         'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
             }
@@ -1072,8 +1076,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        video_tags = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
-        , video_webpage, re.DOTALL | re.IGNORECASE);
+        video_tags = [
+            unescapeHTML(m.group('content'))
+            for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
+
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
@@ -1261,8 +1267,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'title': video_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
-            'tags' : video_tags,
             'categories': video_categories,
+            'tags': video_tags,
             'subtitles': video_subtitles,
             'automatic_captions': automatic_captions,
             'duration': video_duration,

From 5e1eddb939a4fbc2a5ef10111c8141c842cf01d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 21:18:16 +0600
Subject: [PATCH 0917/2145] [youtube] Show info message for multifeed videos
 according to noplaylist option

---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bcd27408d..8a5ef2e70 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1089,22 +1089,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if (not self._downloader.params.get('noplaylist') and
-                not smuggled_data.get('force_singlefeed', False) and
-                'multifeed_metadata_list' in video_info):
-            entries = []
-            multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
-            for feed in multifeed_metadata_list.split(','):
-                feed_data = compat_parse_qs(feed)
-                entries.append({
-                    '_type': 'url_transparent',
-                    'ie_key': 'Youtube',
-                    'url': smuggle_url(
-                        '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
-                        {'force_singlefeed': True}),
-                    'title': '%s (%s)' % (video_title, feed_data['title'][0]),
-                })
-            return self.playlist_result(entries, video_id, video_title, video_description)
+        if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
+            if not self._downloader.params.get('noplaylist'):
+                entries = []
+                feed_ids = []
+                multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+                for feed in multifeed_metadata_list.split(','):
+                    feed_data = compat_parse_qs(feed)
+                    entries.append({
+                        '_type': 'url_transparent',
+                        'ie_key': 'Youtube',
+                        'url': smuggle_url(
+                            '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                            {'force_singlefeed': True}),
+                        'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                    })
+                    feed_ids.append(feed_data['id'][0])
+                self.to_screen(
+                    'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
+                    % (', '.join(feed_ids), video_id))
+                return self.playlist_result(entries, video_id, video_title, video_description)
+            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
         if 'view_count' in video_info:
             view_count = int(video_info['view_count'][0])

From 2711e41bcdaf8e8234dd32e5df6fb76f23179f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Jul 2015 18:47:20 +0200
Subject: [PATCH 0918/2145] Credit slangangular for the sportschau extractor
 (#6199)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e75e9885d..aa6b88cc0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -135,3 +135,4 @@ Bernhard Minks
 sceext
 Zach Bruggeman
 Tjark Saul
+slangangular

From d12a1a47d5ff0833a1ecd7fe47f4ffa67a9b4a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jul 2015 00:55:06 +0600
Subject: [PATCH 0919/2145] [bbc] Improve work with mediaselection URLs

---
 youtube_dl/extractor/bbc.py | 50 ++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 3d9366644..9a1b6e3dc 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -14,12 +14,15 @@ from ..utils import (
 )
 from ..compat import compat_HTTPError
 
+
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+    _MEDIASELECTOR_URLS = [
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
+    ]
 
     _TESTS = [
         {
@@ -161,6 +164,10 @@ class BBCCoUkIE(InfoExtractor):
         }
     ]
 
+    class MediaSelectionError(Exception):
+        def __init__(self, id):
+            self.id = id
+
     def _extract_asx_playlist(self, connection, programme_id):
         asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
         return [ref.get('href') for ref in asx.findall('./Entry/ref')]
@@ -211,8 +218,7 @@ class BBCCoUkIE(InfoExtractor):
     def _extract_medias(self, media_selection):
         error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
         if error is not None:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
+            raise BBCCoUkIE.MediaSelectionError(error.get('id'))
         return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
 
     def _extract_connections(self, media):
@@ -269,17 +275,23 @@ class BBCCoUkIE(InfoExtractor):
             ]
         return subtitles
 
+    def _raise_extractor_error(self, media_selection_error):
+        raise ExtractorError(
+            '%s returned error: %s' % (self.IE_NAME, media_selection_error.id),
+            expected=True)
+
     def _download_media_selector(self, programme_id):
-        try:
-            return self._download_media_selector_url(
-                self._MEDIASELECTOR_URL % programme_id, programme_id)
-        except ExtractorError as e:
-            if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror':
-                 # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/
-                 return self._download_media_selector_url(
-                     self._MEDIASELECTOR_ALT_URL % programme_id, programme_id)
-            else:
-                 raise
+        last_exception = None
+        for mediaselector_url in self._MEDIASELECTOR_URLS:
+            try:
+                return self._download_media_selector_url(
+                    mediaselector_url % programme_id, programme_id)
+            except BBCCoUkIE.MediaSelectionError as e:
+                if e.id == 'notukerror':
+                    last_exception = e
+                    continue
+                self._raise_extractor_error(e)
+        self._raise_extractor_error(last_exception)
 
     def _download_media_selector_url(self, url, programme_id=None):
         try:
@@ -432,10 +444,14 @@ class BBCIE(BBCCoUkIE):
     IE_DESC = 'BBC'
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    # fails with notukerror for some videos ( non news sites such as bbc.com/travel )
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-    # limited selection of formats but may work where the above does not
-    _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+    _MEDIASELECTOR_URLS = [
+        # Provides more formats, namely direct mp4 links, but fails on some videos with
+        # notukerror for non UK (?) users (e.g.
+        # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s',
+        # Provides fewer formats, but works everywhere for everybody (hopefully)
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s',
+    ]
 
     _TESTS = [{
         # article with multiple videos embedded with data-media-meta containing

From 799207e838e0404aaa5cb6658e41bef108aced16 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 29 Jul 2015 23:20:37 +0100
Subject: [PATCH 0920/2145] [viewster] extract the api auth token

Closes #6406.
---
 youtube_dl/compat.py             | 6 ++++++
 youtube_dl/extractor/common.py   | 8 ++++++++
 youtube_dl/extractor/viewster.py | 5 ++++-
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0c57c7aeb..e4b9286c0 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -42,6 +42,11 @@ try:
 except ImportError:  # Python 2
     import cookielib as compat_cookiejar
 
+try:
+    import http.cookies as compat_cookies
+except ImportError:  # Python 2
+    import Cookie as compat_cookies
+
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
@@ -436,6 +441,7 @@ __all__ = [
     'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
+    'compat_cookies',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d54866d1f..dc5080504 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -14,10 +14,12 @@ import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
+    compat_cookies,
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
+    compat_urllib_request,
     compat_urlparse,
     compat_str,
 )
@@ -1074,6 +1076,12 @@ class InfoExtractor(object):
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
+    def _get_cookies(self, url):
+        """ Return a compat_cookies.SimpleCookie with the cookies for the url """
+        req = compat_urllib_request.Request(url)
+        self._downloader.cookiejar.add_cookie_header(req)
+        return compat_cookies.SimpleCookie(req.get_header('Cookie'))
+
     def get_testcases(self, include_onlymatching=False):
         t = getattr(self, '_TEST', None)
         if t:
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 6ef36290b..393b63618 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -62,7 +62,6 @@ class ViewsterIE(InfoExtractor):
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
-    _AUTH_TOKEN = '/YqhSYsx8EaU9Bsta3ojlA=='
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
         request = compat_urllib_request.Request(url)
@@ -72,6 +71,10 @@ class ViewsterIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        # Get 'api_token' cookie
+        self._request_webpage(url, video_id)
+        cookies = self._get_cookies(url)
+        self._AUTH_TOKEN = compat_urllib_parse.unquote(cookies['api_token'].value)
 
         info = self._download_json(
             'https://public-api.viewster.com/search/%s' % video_id,

From 1f04873517e41d2f54fc4e65c46c08ba85b23010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 30 Jul 2015 19:12:37 +0200
Subject: [PATCH 0921/2145] [viewster] Use 'compat_urllib_parse_unquote'

---
 youtube_dl/extractor/viewster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 393b63618..65324d903 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from ..utils import (
     determine_ext,
@@ -74,7 +75,7 @@ class ViewsterIE(InfoExtractor):
         # Get 'api_token' cookie
         self._request_webpage(url, video_id)
         cookies = self._get_cookies(url)
-        self._AUTH_TOKEN = compat_urllib_parse.unquote(cookies['api_token'].value)
+        self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 
         info = self._download_json(
             'https://public-api.viewster.com/search/%s' % video_id,

From 80fb6d4aa47154a1e963b28a17a065dc40a436b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Jul 2015 00:54:26 +0600
Subject: [PATCH 0922/2145] [soundcloud:user] Rework extractor (Closes #6399)

---
 youtube_dl/extractor/soundcloud.py | 131 ++++++++++++++++++++++-------
 1 file changed, 101 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 0a6c9fe72..f1307dc83 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!sets/|(?:likes|tracks)/?(?:$|[?#]))
+                            (?!(?:tracks|sets|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -293,60 +293,131 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|sets|reposts|likes|spotlight)/?)?(\?.*)?$'
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
-        'url': 'https://soundcloud.com/the-concept-band',
+        'url': 'https://soundcloud.com/the-akashic-chronicler',
         'info_dict': {
-            'id': '9615865',
-            'title': 'The Royal Concept',
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (All)',
         },
-        'playlist_mincount': 12
-    }, {
-        'url': 'https://soundcloud.com/the-concept-band/likes',
-        'info_dict': {
-            'id': '9615865',
-            'title': 'The Royal Concept',
-        },
-        'playlist_mincount': 1,
+        'playlist_mincount': 112,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
-        'only_matching': True,
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Tracks)',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Playlists)',
+        },
+        'playlist_mincount': 3,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Reposts)',
+        },
+        'playlist_mincount': 9,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Likes)',
+        },
+        'playlist_mincount': 333,
+    }, {
+        'url': 'https://soundcloud.com/grynpyret/spotlight',
+        'info_dict': {
+            'id': '7098329',
+            'title': 'Grynpyret (Spotlight)',
+        },
+        'playlist_mincount': 1,
     }]
 
+    _API_BASE = 'https://api.soundcloud.com'
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
+
+    _BASE_URL_MAP = {
+        'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
+        'tracks': '%s/users/%%s/tracks' % _API_BASE,
+        'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
+        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
+        'likes': '%s/users/%%s/likes' % _API_V2_BASE,
+        'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
+    }
+
+    _TITLE_MAP = {
+        'all': 'All',
+        'tracks': 'Tracks',
+        'sets': 'Playlists',
+        'reposts': 'Reposts',
+        'likes': 'Likes',
+        'spotlight': 'Spotlight',
+    }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group('user')
-        resource = mobj.group('rsrc')
-        if resource is None:
-            resource = 'tracks'
-        elif resource == 'likes':
-            resource = 'favorites'
 
         url = 'http://soundcloud.com/%s/' % uploader
         resolv_url = self._resolv_url(url)
         user = self._download_json(
             resolv_url, uploader, 'Downloading user info')
-        base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource)
+
+        resource = mobj.group('rsrc') or 'all'
+        base_url = self._BASE_URL_MAP[resource] % user['id']
+
+        next_href = None
 
         entries = []
         for i in itertools.count():
-            data = compat_urllib_parse.urlencode({
-                'offset': i * 50,
-                'limit': 50,
-                'client_id': self._CLIENT_ID,
-            })
-            new_entries = self._download_json(
-                base_url + data, uploader, 'Downloading track page %s' % (i + 1))
-            if len(new_entries) == 0:
+            if not next_href:
+                data = compat_urllib_parse.urlencode({
+                    'offset': i * 50,
+                    'limit': 50,
+                    'client_id': self._CLIENT_ID,
+                    'linked_partitioning': '1',
+                    'representation': 'speedy',
+                })
+                next_href = base_url + '?' + data
+
+            response = self._download_json(
+                next_href, uploader, 'Downloading track page %s' % (i + 1))
+
+            collection = response['collection']
+
+            if not collection:
                 self.to_screen('%s: End page received' % uploader)
                 break
-            entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries)
+
+            def resolve_permalink_url(candidates):
+                for cand in candidates:
+                    if isinstance(cand, dict):
+                        permalink_url = cand.get('permalink_url')
+                        if permalink_url and permalink_url.startswith('http'):
+                            return permalink_url
+
+            for e in collection:
+                permalink_url = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
+                if permalink_url:
+                    entries.append(self.url_result(permalink_url))
+
+            if 'next_href' in response:
+                next_href = response['next_href']
+                if not next_href:
+                    break
+            else:
+                next_href = None
 
         return {
             '_type': 'playlist',
             'id': compat_str(user['id']),
-            'title': user['username'],
+            'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
             'entries': entries,
         }
 

From 09b6468d3027bbc0180cd8b10aa848180cae4005 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 31 Jul 2015 21:28:59 +0100
Subject: [PATCH 0923/2145] [vidme] extract the uploader

---
 youtube_dl/extractor/vidme.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index e0b55078b..575a62f3a 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -50,6 +50,7 @@ class VidmeIE(InfoExtractor):
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
+        uploader = self._html_search_regex('class="video_author_username">([^<]+)', webpage, 'uploader', fatal=False)
 
         return {
             'id': video_id,
@@ -63,4 +64,5 @@ class VidmeIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
+            'uploader': uploader,
         }

From 16a089780e87d57b5295c4dfb8da1217ef6eda70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 03:50:03 +0600
Subject: [PATCH 0924/2145] [soundcloud] Clean up _VALID_URLs

---
 youtube_dl/extractor/soundcloud.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index f1307dc83..6ce86cbcd 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!(?:tracks|sets|reposts|likes|spotlight)/?(?:$|[?#]))
+                            (?!(?:tracks|sets(?:/[^/?#]+)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -293,7 +293,15 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|sets|reposts|likes|spotlight)/?)?(\?.*)?$'
+    _VALID_URL = r'''(?x)
+                        https?://
+                            (?:(?:www|m)\.)?soundcloud\.com/
+                            (?P<user>[^/]+)
+                            (?:/
+                                (?P<rsrc>tracks|sets|reposts|likes|spotlight)
+                            )?
+                            /?(?:[?#].*)?$
+                    '''
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-akashic-chronicler',

From 3b58d94f71b0e7b3ace6f5965f7335aa95fd0c1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 04:03:52 +0600
Subject: [PATCH 0925/2145] [vidme] Make uploader optional and add test

---
 youtube_dl/extractor/vidme.py | 37 ++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 575a62f3a..157bb74fe 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -22,6 +22,27 @@ class VidmeIE(InfoExtractor):
             'timestamp': 1406313244,
             'upload_date': '20140725',
             'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+    }, {
+        # tests uploader field
+        'url': 'https://vid.me/4Iib',
+        'info_dict': {
+            'id': '4Iib',
+            'ext': 'mp4',
+            'title': 'The Carver',
+            'description': 'md5:e9c24870018ae8113be936645b93ba3c',
+            'duration': 97.859999999999999,
+            'timestamp': 1433203629,
+            'upload_date': '20150602',
+            'uploader': 'Thomas',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
@@ -40,17 +61,23 @@ class VidmeIE(InfoExtractor):
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage, default='')
         thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = int_or_none(self._og_search_property('updated_time', webpage, fatal=False))
-        width = int_or_none(self._og_search_property('video:width', webpage, fatal=False))
-        height = int_or_none(self._og_search_property('video:height', webpage, fatal=False))
+        timestamp = int_or_none(self._og_search_property(
+            'updated_time', webpage, fatal=False))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, fatal=False))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, fatal=False))
         duration = float_or_none(self._html_search_regex(
             r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
         view_count = str_to_int(self._html_search_regex(
-            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
+            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?',
+            webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
-        uploader = self._html_search_regex('class="video_author_username">([^<]+)', webpage, 'uploader', fatal=False)
+        uploader = self._html_search_regex(
+            'class="video_author_username"[^>]*>([^<]+)',
+            webpage, 'uploader', default=None)
 
         return {
             'id': video_id,

From 30a453884e6a4d228ced4cdddcfa8e79a768755a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 31 Jul 2015 14:41:30 +0100
Subject: [PATCH 0926/2145] [viewster] use head request to extract api token

Closes #6419.
---
 youtube_dl/extractor/viewster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 65324d903..cda02ba24 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -11,6 +11,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     parse_iso8601,
+    HEADRequest,
 )
 
 
@@ -73,7 +74,7 @@ class ViewsterIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # Get 'api_token' cookie
-        self._request_webpage(url, video_id)
+        self._request_webpage(HEADRequest(url), video_id)
         cookies = self._get_cookies(url)
         self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 

From 44cae2fb2e7b8cffcdfdd57b1224115345009cc5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 30 Jul 2015 23:55:57 +0100
Subject: [PATCH 0927/2145] [gdcvault] Add support for audio extraction (fixes
 #5784)

Closes #6412.
---
 youtube_dl/extractor/gdcvault.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 43f916412..c3017d451 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -7,7 +7,10 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
-from ..utils import remove_end
+from ..utils import (
+    remove_end,
+    HEADRequest,
+)
 
 
 class GDCVaultIE(InfoExtractor):
@@ -73,10 +76,20 @@ class GDCVaultIE(InfoExtractor):
         return video_formats
 
     def _parse_flv(self, xml_description):
-        video_formats = []
+        formats = []
         akamai_url = xml_description.find('./metadata/akamaiHost').text
+        audios = xml_description.find('./metadata/audios')
+        if audios is not None:
+            for audio in audios:
+                formats.append({
+                    'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                    'play_path': remove_end(audio.get('url'), '.flv'),
+                    'ext': 'flv',
+                    'vcodec': 'none',
+                    'format_id': audio.get('code'),
+                })
         slide_video_path = xml_description.find('./metadata/slideVideo').text
-        video_formats.append({
+        formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(slide_video_path, '.flv'),
             'ext': 'flv',
@@ -86,7 +99,7 @@ class GDCVaultIE(InfoExtractor):
             'format_id': 'slides',
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
-        video_formats.append({
+        formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(speaker_video_path, '.flv'),
             'ext': 'flv',
@@ -95,7 +108,7 @@ class GDCVaultIE(InfoExtractor):
             'preference': -1,
             'format_id': 'speaker',
         })
-        return video_formats
+        return formats
 
     def _login(self, webpage_url, display_id):
         (username, password) = self._get_login_info()
@@ -133,16 +146,18 @@ class GDCVaultIE(InfoExtractor):
             r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
             start_page, 'url', default=None)
         if direct_url:
-            video_url = 'http://www.gdcvault.com/' + direct_url
             title = self._html_search_regex(
                 r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>',
                 start_page, 'title')
+            video_url = 'http://www.gdcvault.com' + direct_url
+            # resolve the url so that we can detect the correct extension
+            head = self._request_webpage(HEADRequest(video_url), video_id)
+            video_url = head.geturl()
 
             return {
                 'id': video_id,
                 'display_id': display_id,
                 'url': video_url,
-                'ext': 'flv',
                 'title': title,
             }
 

From 525a87f58ee9c4ee91b5b0384184dabf6d87eef3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 1 Aug 2015 11:40:34 +0200
Subject: [PATCH 0928/2145] [gdcvault] Fix typo: xml_decription_url ->
 xml_description_url

---
 youtube_dl/extractor/gdcvault.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index c3017d451..a6834db43 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -183,8 +183,8 @@ class GDCVaultIE(InfoExtractor):
             # Fallback to the older format
             xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
 
-        xml_decription_url = xml_root + 'xml/' + xml_name
-        xml_description = self._download_xml(xml_decription_url, display_id)
+        xml_description_url = xml_root + 'xml/' + xml_name
+        xml_description = self._download_xml(xml_description_url, display_id)
 
         video_title = xml_description.find('./metadata/title').text
         video_formats = self._parse_mp4(xml_description)

From ee114368ad0bb9822449295910263a99f9de4e1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 20:22:13 +0600
Subject: [PATCH 0929/2145] [utils] Make value optional for find_xpath_attr

This allows selecting particular attributes by name but without specifying the value and similar to xpath syntax `[@attrib]`
---
 test/test_utils.py  |  9 +++++++++
 youtube_dl/utils.py | 13 ++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 65692a9fb..a759b2da9 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -235,12 +235,21 @@ class TestUtil(unittest.TestCase):
             <node x="a"/>
             <node x="a" y="c" />
             <node x="b" y="d" />
+            <node x="" />
         </root>'''
         doc = xml.etree.ElementTree.fromstring(testxml)
 
+        self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'n'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'n', 'v'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x'), doc[1])
         self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'b'), doc[3])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'y'), doc[2])
         self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'd'), doc[3])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x', ''), doc[4])
 
     def test_xpath_with_ns(self):
         testxml = '''<root xmlns:media="http://example.com/">
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 88f9f9070..78dc2b449 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -139,21 +139,24 @@ def write_json_file(obj, fn):
 
 
 if sys.version_info >= (2, 7):
-    def find_xpath_attr(node, xpath, key, val):
+    def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z-]+$', key)
-        assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
-        expr = xpath + "[@%s='%s']" % (key, val)
+        if val:
+            assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
+        expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:
-    def find_xpath_attr(node, xpath, key, val):
+    def find_xpath_attr(node, xpath, key, val=None):
         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
         # .//node does not match if a node is a direct child of . !
         if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
 
         for f in node.findall(xpath):
-            if f.attrib.get(key) == val:
+            if key not in f.attrib:
+                continue
+            if val is None or f.attrib.get(key) == val:
                 return f
         return None
 

From 3f125c8c70e8109bc90d4446b40740133e343b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 21:43:33 +0600
Subject: [PATCH 0930/2145] [nbcnews] Extend _VALID_URL

---
 youtube_dl/extractor/nbc.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index dc2091be0..ccdbfb6c9 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -124,7 +124,7 @@ class NBCSportsIE(InfoExtractor):
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
-        (?:feature|nightly-news)/[^/]+/(?P<title>.+))
+        (?:watch|feature|nightly-news)/[^/]+/(?P<title>.+))
         '''
 
     _TESTS = [
@@ -169,6 +169,10 @@ class NBCNewsIE(InfoExtractor):
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 55eae65b39d754d699ad9de3f9c99fcdf62e0176 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Aug 2015 00:42:23 +0800
Subject: [PATCH 0931/2145] Credit @cyb3r for the ir90tv extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index aa6b88cc0..d16d34272 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -136,3 +136,4 @@ sceext
 Zach Bruggeman
 Tjark Saul
 slangangular
+Behrouz Abbasi

From a107193e4b7a3d5414dd7422263c34ac0e309ec4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:13:21 +0600
Subject: [PATCH 0932/2145] [extractor/common] Extract f4m and m3u8 formats,
 subtitles and info

---
 youtube_dl/extractor/common.py | 200 ++++++++++++++++++++++++---------
 1 file changed, 149 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..f9578b838 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -18,6 +18,7 @@ from ..compat import (
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
@@ -37,6 +38,7 @@ from ..utils import (
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
+    url_basename,
 )
 
 
@@ -978,69 +980,165 @@ class InfoExtractor(object):
         self._sort_formats(formats)
         return formats
 
-    # TODO: improve extraction
-    def _extract_smil_formats(self, smil_url, video_id, fatal=True):
-        smil = self._download_xml(
-            smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file', fatal=fatal)
+    @staticmethod
+    def _xpath_ns(path, namespace=None):
+        if not namespace:
+            return path
+        out = []
+        for c in path.split('/'):
+            if not c or c == '.':
+                out.append(c)
+            else:
+                out.append('{%s}%s' % (namespace, c))
+        return '/'.join(out)
+
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+
         if smil is False:
             assert not fatal
             return []
 
-        base = smil.find('./head/meta').get('base')
+        namespace = self._search_regex(
+            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+
+        return self._parse_smil_formats(
+            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
+
+    def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+        if smil is False:
+            return {}
+        return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
+
+    def _download_smil(self, smil_url, video_id, fatal=True):
+        return self._download_xml(
+            smil_url, video_id, 'Downloading SMIL file',
+            'Unable to download SMIL file', fatal=fatal)
+
+    def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
+        namespace = self._search_regex(
+            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+
+        formats = self._parse_smil_formats(
+            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
+        subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
+
+        video_id = os.path.splitext(url_basename(smil_url))[0]
+        title = None
+        description = None
+        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
+            name = meta.attrib.get('name')
+            content = meta.attrib.get('content')
+            if not name or not content:
+                continue
+            if not title and name == 'title':
+                title = content
+            elif not description and name in ('description', 'abstract'):
+                description = content
+
+        return {
+            'id': video_id,
+            'title': title or video_id,
+            'description': description,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
+        base = smil_url
+        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
+            b = meta.get('base') or meta.get('httpBase')
+            if b:
+                base = b
+                break
 
         formats = []
         rtmp_count = 0
-        if smil.findall('./body/seq/video'):
-            video = smil.findall('./body/seq/video')[0]
-            fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
-            formats.extend(fmts)
-        else:
-            for video in smil.findall('./body/switch/video'):
-                fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
-                formats.extend(fmts)
+        http_count = 0
+
+        videos = smil.findall(self._xpath_ns('.//video', namespace))
+        for video in videos:
+            src = video.get('src')
+            if not src:
+                continue
+
+            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            filesize = int_or_none(video.get('size') or video.get('fileSize'))
+            width = int_or_none(video.get('width'))
+            height = int_or_none(video.get('height'))
+            proto = video.get('proto')
+            ext = video.get('ext')
+            src_ext = determine_ext(src)
+            streamer = video.get('streamer') or base
+
+            if proto == 'rtmp' or streamer.startswith('rtmp'):
+                rtmp_count += 1
+                formats.append({
+                    'url': streamer,
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                    'tbr': bitrate,
+                    'filesize': filesize,
+                    'width': width,
+                    'height': height,
+                })
+                continue
+
+            src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+
+            if proto == 'm3u8' or src_ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls'))
+                continue
+
+            if src_ext == 'f4m':
+                f4m_url = src_url
+                if not f4m_params:
+                    f4m_params = {
+                        'hdcore': '3.2.0',
+                        'plugin': 'flowplayer-3.2.0.1',
+                    }
+                f4m_url += '&' if '?' in f4m_url else '?'
+                f4m_url += compat_urllib_parse.urlencode(f4m_params).encode('utf-8')
+                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
+                continue
+
+            if src_url.startswith('http'):
+                http_count += 1
+                formats.append({
+                    'url': src_url,
+                    'ext': ext or src_ext or 'flv',
+                    'format_id': 'http-%d' % (bitrate or http_count),
+                    'tbr': bitrate,
+                    'filesize': filesize,
+                    'width': width,
+                    'height': height,
+                })
+                continue
 
         self._sort_formats(formats)
 
         return formats
 
-    def _parse_smil_video(self, video, video_id, base, rtmp_count):
-        src = video.get('src')
-        if not src:
-            return [], rtmp_count
-        bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-        width = int_or_none(video.get('width'))
-        height = int_or_none(video.get('height'))
-        proto = video.get('proto')
-        if not proto:
-            if base:
-                if base.startswith('rtmp'):
-                    proto = 'rtmp'
-                elif base.startswith('http'):
-                    proto = 'http'
-        ext = video.get('ext')
-        if proto == 'm3u8':
-            return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
-        elif proto == 'rtmp':
-            rtmp_count += 1
-            streamer = video.get('streamer') or base
-            return ([{
-                'url': streamer,
-                'play_path': src,
-                'ext': 'flv',
-                'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
-                'tbr': bitrate,
-                'width': width,
-                'height': height,
-            }], rtmp_count)
-        elif proto.startswith('http'):
-            return ([{
-                'url': base + src,
-                'ext': ext or 'flv',
-                'tbr': bitrate,
-                'width': width,
-                'height': height,
-            }], rtmp_count)
+    def _parse_smil_subtitles(self, smil, namespace=None):
+        subtitles = {}
+        for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
+            src = textstream.get('src')
+            if not src:
+                continue
+            ext = textstream.get('ext') or determine_ext(src)
+            if not ext:
+                type_ = textstream.get('type')
+                if type_ == 'text/srt':
+                    ext = 'srt'
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName')
+            subtitles.setdefault(lang, []).append({
+                'url': src,
+                'ext': ext,
+            })
+        return subtitles
 
     def _live_title(self, name):
         """ Generate the title for a live video """

From e5e8d20a3a65832c74b002f247866fcbb92e9246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:13:59 +0600
Subject: [PATCH 0933/2145] [extractor/generic] Improve generic SMIL detection

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8cef61c3c..6900ed96f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1110,11 +1110,13 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed?
+        # Is it an RSS feed or a SMIL file?
         try:
             doc = parse_xml(webpage)
             if doc.tag == 'rss':
                 return self._extract_rss(url, video_id, doc)
+            elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
+                return self._parse_smil(doc, url, video_id)
         except compat_xml_parse_error:
             pass
 

From 308cfe0ab3ec7122602ba2d6a4e3acd2caa7a757 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:14:41 +0600
Subject: [PATCH 0934/2145] [test_downloader] Respect --force-generic-extractor

---
 test/test_download.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 1110357a7..284418834 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -136,7 +136,9 @@ def generator(test_case):
                     # We're not using .download here sine that is just a shim
                     # for outside error handling, and returns the exit code
                     # instead of the result dict.
-                    res_dict = ydl.extract_info(test_case['url'])
+                    res_dict = ydl.extract_info(
+                        test_case['url'],
+                        force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):

From 645f814544f9d40386e504a1eb8cf3558f2c109e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:15:33 +0600
Subject: [PATCH 0935/2145] [test/helper] Allow dicts for mincount

---
 test/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e1129e58f..c8b34654d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -133,8 +133,8 @@ def expect_info_dict(self, got_dict, expected_dict):
             elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
                 got = got_dict.get(info_field)
                 self.assertTrue(
-                    isinstance(got, list),
-                    'Expected field %s to be a list, but it is of type %s' % (
+                    isinstance(got, (list, dict)),
+                    'Expected field %s to be a list or a dict, but it is of type %s' % (
                         info_field, type(got).__name__))
                 expected_num = int(expected.partition(':')[2])
                 assertGreaterEqual(

From 8765222d2211cd6f2a40611249181af0bbb2d531 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:16:21 +0600
Subject: [PATCH 0936/2145] [extractor/generic] Add generic SMIL tests

---
 youtube_dl/extractor/generic.py | 68 +++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6900ed96f..27584c44c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -130,6 +130,74 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
+        # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
+        {
+            'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
+            'info_dict': {
+                'id': 'smil',
+                'ext': 'mp4',
+                'title': 'Automatics, robotics and biocybernetics',
+                'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+                'formats': 'mincount:16',
+                'subtitles': 'mincount:1',
+            },
+            'params': {
+                'force_generic_extractor': True,
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
+        {
+            'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
+            'info_dict': {
+                'id': 'hds',
+                'ext': 'flv',
+                'title': 'hds',
+                'formats': 'mincount:1',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from https://www.restudy.dk/video/play/id/1637
+        {
+            'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
+            'info_dict': {
+                'id': 'video_1637',
+                'ext': 'flv',
+                'title': 'video_1637',
+                'formats': 'mincount:3',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
+        {
+            'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
+            'info_dict': {
+                'id': 'smil-service',
+                'ext': 'flv',
+                'title': 'smil-service',
+                'formats': 'mincount:1',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
+        {
+            'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
+            'info_dict': {
+                'id': '4719370',
+                'ext': 'mp4',
+                'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
+                'formats': 'mincount:3',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 41c3a5a7beebbf5f60c5edb5093d564f0829c5c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:20:49 +0600
Subject: [PATCH 0937/2145] [extractor/common] Fix python 3

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f9578b838..c123d9fca 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1101,7 +1101,7 @@ class InfoExtractor(object):
                         'plugin': 'flowplayer-3.2.0.1',
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
-                f4m_url += compat_urllib_parse.urlencode(f4m_params).encode('utf-8')
+                f4m_url += compat_urllib_parse.urlencode(f4m_params)
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
                 continue
 

From 17712eeb1933f53696c1fc53606174e988a96472 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:31:17 +0600
Subject: [PATCH 0938/2145] [extractor/common] Extract namespace parse routine

---
 youtube_dl/extractor/common.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c123d9fca..717dcec7b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -999,8 +999,7 @@ class InfoExtractor(object):
             assert not fatal
             return []
 
-        namespace = self._search_regex(
-            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+        namespace = self._parse_smil_namespace(smil)
 
         return self._parse_smil_formats(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
@@ -1017,8 +1016,7 @@ class InfoExtractor(object):
             'Unable to download SMIL file', fatal=fatal)
 
     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
-        namespace = self._search_regex(
-            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+        namespace = self._parse_smil_namespace(smil)
 
         formats = self._parse_smil_formats(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
@@ -1045,6 +1043,10 @@ class InfoExtractor(object):
             'subtitles': subtitles,
         }
 
+    def _parse_smil_namespace(self, smil):
+        return self._search_regex(
+            r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
+
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
         base = smil_url
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):

From fa7a1cc5ef52a8dd9a355ab37a74be55ac2ddc1f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 22 Jul 2015 12:34:42 +0100
Subject: [PATCH 0939/2145] [screenwavemedia] fix info extraction (fixes #6270)

Closes #6330.
---
 youtube_dl/extractor/screenwavemedia.py | 84 +++++++++++--------------
 1 file changed, 36 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index d1ab66b32..09c085dcf 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -1,12 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    js_to_json,
 )
 
 
@@ -22,59 +21,48 @@ class ScreenwaveMediaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         playerdata = self._download_webpage(
-            'http://player.screenwavemedia.com/play/player.php?id=%s' % video_id,
+            'http://player.screenwavemedia.com/player.php?id=%s' % video_id,
             video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
             r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
-        vidurl = self._search_regex(
-            r'\'vidurl\'\s*:\s*"([^"]+)"', playerdata, 'vidurl').replace('\\/', '/')
 
-        videolist_url = None
+        playerconfig = self._download_webpage(
+            'http://player.screenwavemedia.com/player.js',
+            video_id, 'Downloading playerconfig webpage')
 
-        mobj = re.search(r"'videoserver'\s*:\s*'(?P<videoserver>[^']+)'", playerdata)
-        if mobj:
-            videoserver = mobj.group('videoserver')
-            mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
-            vidid = mobj.group('vidid') if mobj else video_id
-            videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
-        else:
-            mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
-            if mobj:
-                videolist_url = mobj.group('smil')
+        videoserver = self._search_regex(r"'videoserver'\s*:\s*'([^']+)", playerconfig, 'videoserver')
 
-        if videolist_url:
-            videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
-            formats = []
-            baseurl = vidurl[:vidurl.rfind('/') + 1]
-            for video in videolist.findall('.//video'):
-                src = video.get('src')
-                if not src:
-                    continue
-                file_ = src.partition(':')[-1]
-                width = int_or_none(video.get('width'))
-                height = int_or_none(video.get('height'))
-                bitrate = int_or_none(video.get('system-bitrate'), scale=1000)
-                format = {
-                    'url': baseurl + file_,
-                    'format_id': src.rpartition('.')[0].rpartition('_')[-1],
-                }
-                if width or height:
-                    format.update({
-                        'tbr': bitrate,
-                        'width': width,
-                        'height': height,
-                    })
-                else:
-                    format.update({
-                        'abr': bitrate,
-                        'vcodec': 'none',
-                    })
-                formats.append(format)
-        else:
-            formats = [{
-                'url': vidurl,
-            }]
+        sources = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                    'sources',
+                ).replace(
+                    "' + thisObj.options.videoserver + '",
+                    videoserver
+                ).replace(
+                    "' + playerVidId + '",
+                    video_id
+                )
+            ),
+            video_id
+        )
+
+        formats = []
+        for source in sources:
+            if source['type'] == 'hls':
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+            else:
+                format_label = source.get('label')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_label, 'height', default=None))
+                formats.append({
+                    'url': source['file'],
+                    'format': format_label,
+                    'ext': source.get('type'),
+                    'height': height,
+                })
         self._sort_formats(formats)
 
         return {

From 9cc93c64aa321260475a2bdf7d8626cdd16bf8ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 1 Aug 2015 22:15:43 +0200
Subject: [PATCH 0940/2145] [screenwavemedia] Use the IP for the videoserver
 (fixes #6397)

For http://cinemassacre.com/2015/07/28/avgn-seaman-for-dreamcast/ the other server returns a 403 error.
---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 09c085dcf..3bc84989e 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -31,7 +31,7 @@ class ScreenwaveMediaIE(InfoExtractor):
             'http://player.screenwavemedia.com/player.js',
             video_id, 'Downloading playerconfig webpage')
 
-        videoserver = self._search_regex(r"'videoserver'\s*:\s*'([^']+)", playerconfig, 'videoserver')
+        videoserver = self._search_regex(r"\[ipaddress\]\s*=>\s*([\d\.]+)", playerdata, 'videoserver')
 
         sources = self._parse_json(
             js_to_json(

From cdc682d5a467b7188eb13b5eeb76eb5dd544d1f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 04:21:16 +0600
Subject: [PATCH 0941/2145] [nowtv] Fix extraction (Closes #6357)

---
 youtube_dl/extractor/nowtv.py | 63 +++++++++++++++--------------------
 1 file changed, 27 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 0b5ff4760..de6bc6d96 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     ExtractorError,
+    determine_ext,
     int_or_none,
     parse_iso8601,
     parse_duration,
@@ -15,7 +16,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?P<station>rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
 
     _TESTS = [{
         # rtl
@@ -23,7 +24,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203519',
             'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Die neuen Bauern und eine Hochzeit',
             'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -32,7 +33,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2786,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -41,7 +42,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203481',
             'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Berlin - Tag & Nacht (Folge 934)',
             'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -50,7 +51,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2641,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -59,7 +60,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '165780',
             'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hals- und Beinbruch',
             'description': 'md5:b50d248efffe244e6f56737f0911ca57',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -68,7 +69,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2742,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -77,7 +78,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '99205',
             'display_id': 'medicopter-117/angst',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Angst!',
             'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -86,7 +87,7 @@ class NowTVIE(InfoExtractor):
             'duration': 3025,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -95,7 +96,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203521',
             'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
             'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -104,7 +105,7 @@ class NowTVIE(InfoExtractor):
             'duration': 1083,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -113,7 +114,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '128953',
             'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': "Büro-Fall / Chihuahua 'Joel'",
             'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -122,15 +123,13 @@ class NowTVIE(InfoExtractor):
             'duration': 3092,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-        station = mobj.group('station')
+        display_id = self._match_id(url)
 
         info = self._download_json(
             'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
@@ -148,29 +147,19 @@ class NowTVIE(InfoExtractor):
                 raise ExtractorError(
                     'Video %s is not available for free' % video_id, expected=True)
 
-        f = info.get('format', {})
-        station = f.get('station') or station
-
-        STATIONS = {
-            'rtl': 'rtlnow',
-            'rtl2': 'rtl2now',
-            'vox': 'voxnow',
-            'nitro': 'rtlnitronow',
-            'ntv': 'n-tvnow',
-            'superrtl': 'superrtlnow'
-        }
-
         formats = []
         for item in files['items']:
-            item_path = remove_start(item['path'], '/')
-            tbr = int_or_none(item['bitrate'])
-            m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)
-            m3u8_url = m3u8_url.replace('now/', 'now/videos/')
+            if determine_ext(item['path']) != 'f4v':
+                continue
+            app, play_path = remove_start(item['path'], '/').split('/', 1)
             formats.append({
-                'url': m3u8_url,
-                'format_id': '%s-%sk' % (item['id'], tbr),
-                'ext': 'mp4',
-                'tbr': tbr,
+                'url': 'rtmpe://fms.rtl.de',
+                'app': app,
+                'play_path': 'mp4:%s' % play_path,
+                'ext': 'flv',
+                'page_url': url,
+                'player_url': 'http://rtl-now.rtl.de/includes/nc_player.swf',
+                'tbr': int_or_none(item.get('bitrate')),
             })
         self._sort_formats(formats)
 
@@ -178,6 +167,8 @@ class NowTVIE(InfoExtractor):
         description = info.get('articleLong') or info.get('articleShort')
         timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
         duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
         thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
 
         return {

From e422d7f4f78994de8483d2207ab4e00174a2408c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 04:26:59 +0600
Subject: [PATCH 0942/2145] [nowtv] Expand _VALID_URL

---
 youtube_dl/extractor/nowtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index de6bc6d96..11ce37168 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -126,6 +126,9 @@ class NowTVIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From d41d04c0f513ad3b83ab6aee60cf2201710b6063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 06:35:35 +0600
Subject: [PATCH 0943/2145] [videolectures] Fix _VALID_URL

---
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index d6a7eb203..24584dc80 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)(?:/?[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
     _TEST = {

From 5c45bbe57bd791debfd64052ab030298a7c6b718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Aug 2015 15:19:30 +0200
Subject: [PATCH 0944/2145] [nowtv] Remove unused import

---
 youtube_dl/extractor/nowtv.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 11ce37168..ad938fb62 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (

From 25a4c5a9ed59eca0241922363e83e61172527658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Aug 2015 15:19:57 +0200
Subject: [PATCH 0945/2145] [dailymotion:playlist] Use an iterator for the
 entries

So that using '--playlist-end' only downloads the required pages (reported in #2175).
---
 youtube_dl/extractor/dailymotion.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 85d945509..2d90b2224 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -15,7 +15,6 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
-    orderedSet,
     parse_iso8601,
     str_to_int,
     unescapeHTML,
@@ -278,7 +277,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     }]
 
     def _extract_entries(self, id):
-        video_ids = []
+        video_ids = set()
         processed_urls = set()
         for pagenum in itertools.count(1):
             page_url = self._PAGE_TEMPLATE % (id, pagenum)
@@ -291,12 +290,13 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
             processed_urls.add(urlh.geturl())
 
-            video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
+            for video_id in re.findall(r'data-xid="(.+?)"', webpage):
+                if video_id not in video_ids:
+                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
+                    video_ids.add(video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
                 break
-        return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
-                for video_id in orderedSet(video_ids)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From d7d2a9a3dbf1cef78c5085a4aab5d2f336c64cff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 03:28:04 +0100
Subject: [PATCH 0946/2145] [utils] restart download if server does not support
 byte ranges

---
 youtube_dl/downloader/http.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b7f144af9..b2e82cfde 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -57,6 +57,20 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
+
+                if resume_len > 0:
+                    content_range = data.headers.get('Content-Range')
+                    if content_range:
+                        content_range_m = re.search(r'bytes (\d+)-', content_range)
+                        if content_range_m:
+                            # Content-Range is correct - go on
+                            if resume_len == int(content_range_m.group(1)):
+                                break
+
+                    # Content-Range is invalid - wipe the file and do entire redownload
+                    resume_len = 0
+                    open_mode = 'wb'
+
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:

From 8d5b8b477e4b1051482b21ea451f0de1ce23bce7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 03:58:02 +0100
Subject: [PATCH 0947/2145] [utils] import re

---
 youtube_dl/downloader/http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b2e82cfde..f796ee113 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -4,6 +4,7 @@ import errno
 import os
 import socket
 import time
+import re
 
 from .common import FileDownloader
 from ..compat import (

From c3124c3085e6a9a83ee31ace3a7d528a324c42da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:25:08 +0600
Subject: [PATCH 0948/2145] [downloader/http] Simplify

---
 youtube_dl/downloader/http.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f796ee113..0862e90bb 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,20 +58,16 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
-
                 if resume_len > 0:
                     content_range = data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
-                        if content_range_m:
-                            # Content-Range is correct - go on
-                            if resume_len == int(content_range_m.group(1)):
-                                break
-
+                        # Content-Range is correct - go on
+                    if content_range_m and resume_len == int(content_range_m.group(1)):
+                        break
                     # Content-Range is invalid - wipe the file and do entire redownload
                     resume_len = 0
                     open_mode = 'wb'
-
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:

From 10eaa8ef1d2a9699052af9262aa472456548e99b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:25:40 +0600
Subject: [PATCH 0949/2145] [downloader/http] Report unable to resume

---
 youtube_dl/downloader/http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 0862e90bb..2f8490f02 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -66,6 +66,7 @@ class HttpFD(FileDownloader):
                     if content_range_m and resume_len == int(content_range_m.group(1)):
                         break
                     # Content-Range is invalid - wipe the file and do entire redownload
+                    self.report_unable_to_resume()
                     resume_len = 0
                     open_mode = 'wb'
                 break

From 84bc4dcb0f678f0a8c9f993e101b9769e3959f76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:27:47 +0600
Subject: [PATCH 0950/2145] [downloader/http] Clarify rationale for
 Content-Range check (#6426)

---
 youtube_dl/downloader/http.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2f8490f02..a29f5cf31 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,14 +58,21 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
+                # When trying to resume, Content-Range HTTP header of response has to be checked
+                # to match the value of requested Range HTTP header. This is due to a webservers
+                # that don't support resuming and serve a whole file with no Content-Range
+                # set in response despite of requested Range (see
+                # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
                 if resume_len > 0:
                     content_range = data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
-                        # Content-Range is correct - go on
-                    if content_range_m and resume_len == int(content_range_m.group(1)):
-                        break
-                    # Content-Range is invalid - wipe the file and do entire redownload
+                        # Content-Range is present and matches requested Range, resume is possible
+                        if content_range_m and resume_len == int(content_range_m.group(1)):
+                            break
+                    # Content-Range is either not present or invalid. Assuming remote webserver is
+                    # trying to send the whole file, resume is not possible, so wiping the local file
+                    # and performing entire redownload
                     self.report_unable_to_resume()
                     resume_len = 0
                     open_mode = 'wb'

From 754e70cf3e74218ae5d840985fbf07bbe274332a Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sun, 2 Aug 2015 19:21:10 +0100
Subject: [PATCH 0951/2145] [pornhub] Fix video url regular expression.

PornHub seems to have subtly changed their JavaScript. Before, video URL strings were embedded directly in the video's `flashvars_*` object, but they are now assigned to variables of the form `player_quality_*`, which are then added to this object later under the relevant quality key.
---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 0b7886840..fbaa830d6 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -81,7 +81,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"var player_quality_[0-9]{3}p = '([^']+)'", webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From 524229a2975c20887a9a71cae77132e775003537 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:41:17 +0600
Subject: [PATCH 0952/2145] [pornhub] Improve

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fbaa830d6..fec493046 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -81,7 +81,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"var player_quality_[0-9]{3}p = '([^']+)'", webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"player_quality_[0-9]{3}p\s*=\s*'([^']+)'", webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From 51a575159a5a83e4477b03544f419dcf2e9ff0fa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 22:52:12 +0100
Subject: [PATCH 0953/2145] [facebook] extract uploader

---
 youtube_dl/extractor/facebook.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index e17bb9aea..734de4da2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -17,6 +17,8 @@ from ..utils import (
     int_or_none,
     limit_length,
     urlencode_postdata,
+    get_element_by_id,
+    clean_html,
 )
 
 
@@ -161,6 +163,7 @@ class FacebookIE(InfoExtractor):
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id
+        uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
 
         return {
             'id': video_id,
@@ -168,4 +171,5 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'duration': int_or_none(video_data.get('video_duration')),
             'thumbnail': video_data.get('thumbnail_src'),
+            'uploader': uploader,
         }

From 67b8a28a2f69764259cf2e90c0a3785c05c55551 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 3 Aug 2015 00:09:21 +0100
Subject: [PATCH 0954/2145] [facebook] add uploader value to the tests

---
 youtube_dl/extractor/facebook.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 734de4da2..178a7ca4c 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -44,6 +44,7 @@ class FacebookIE(InfoExtractor):
             'id': '637842556329505',
             'ext': 'mp4',
             'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
+            'uploader': 'Tennis on Facebook',
         }
     }, {
         'note': 'Video without discernible title',
@@ -52,6 +53,7 @@ class FacebookIE(InfoExtractor):
             'id': '274175099429670',
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
+            'uploader': 'Asif Nawab Butt',
         },
         'expected_warnings': [
             'title'

From 8de922724b8f3ad31ff7249799de371ff8a5c3ad Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 05:36:17 +0600
Subject: [PATCH 0955/2145] [README.md] Clarify using cookies

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index ac54d7b67..2db3139ee 100644
--- a/README.md
+++ b/README.md
@@ -439,6 +439,12 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
     youtube-dl -- -wNyEUrxzFU
     youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
 
+### How do I pass cookies to youtube-dl?
+
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that cookies file must be in Mozilla/Netscape format and the first line of cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in cookies file and convert newlines if necessary to correspond your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+
+Passing cookies to youtube-dl is a good way to workaround login when particular extractor does not implement it explicitly.
+
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
 As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.

From 47a8b7c14a085ce558db3b5a85ded850cd5df642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 3 Aug 2015 12:00:08 +0200
Subject: [PATCH 0956/2145] [mdr] Change XPath to make it work in python 2.6
 (fixes #6443)

The 'progressiveDownloadUrl' element is a direct child, so they should be equivalent.
---
 youtube_dl/extractor/mdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 5fdd19027..fc7499958 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -29,7 +29,7 @@ class MDRIE(InfoExtractor):
         doc = self._download_xml(domain + xmlurl, video_id)
         formats = []
         for a in doc.findall('./assets/asset'):
-            url_el = a.find('.//progressiveDownloadUrl')
+            url_el = a.find('./progressiveDownloadUrl')
             if url_el is None:
                 continue
             abr = int(a.find('bitrateAudio').text) // 1000

From 8f5639afcbb967f276fb8b35a24559cdcc3b6d32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 19:37:48 +0600
Subject: [PATCH 0957/2145] [pornhub] Improve video quality regex

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fec493046..7b0cdc41a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -94,7 +94,7 @@ class PornHubIE(InfoExtractor):
             format = path.split('/')[5].split('_')[:2]
             format = "-".join(format)
 
-            m = re.match(r'^(?P<height>[0-9]+)P-(?P<tbr>[0-9]+)K$', format)
+            m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
             if m is None:
                 height = None
                 tbr = None

From e704f87f869b98bbed56d7dd0fe27710306c8272 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@nand.wakku.to>
Date: Mon, 3 Aug 2015 01:54:21 +0200
Subject: [PATCH 0958/2145] [twitch] Parse start_time from 't' (closes #6441)

Eg. for VOD links like http://www.twitch.tv/gamesdonequick/v/9136645?t=14h29m15s
---
 youtube_dl/extractor/twitch.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 73ce335b7..a2b6a35aa 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -7,12 +7,15 @@ import random
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_str,
     compat_urllib_parse,
+    compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    parse_duration,
     parse_iso8601,
 )
 
@@ -185,7 +188,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_SHORTCUT = 'v'
 
     _TEST = {
-        'url': 'http://www.twitch.tv/riotgames/v/6528877',
+        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
         'info_dict': {
             'id': 'v6528877',
             'ext': 'mp4',
@@ -197,6 +200,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             'uploader': 'Riot Games',
             'uploader_id': 'riotgames',
             'view_count': int,
+            'start_time': 310,
         },
         'params': {
             # m3u8 download
@@ -216,6 +220,12 @@ class TwitchVodIE(TwitchItemBaseIE):
             item_id, 'mp4')
         self._prefer_source(formats)
         info['formats'] = formats
+
+        parsed_url = compat_urllib_parse_urlparse(url)
+        query = compat_parse_qs(parsed_url.query)
+        if 't' in query:
+            info['start_time'] = parse_duration(query['t'][0])
+
         return info
 
 

From d96d604e5311628ece0234733dbbfe73a58c8d18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 3 Aug 2015 23:04:11 +0200
Subject: [PATCH 0959/2145] YoutubeDL: format spec: don't accept a bare '/'
 (#6124)

---
 test/test_YoutubeDL.py  | 1 +
 youtube_dl/YoutubeDL.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 20f45f439..9a3c28f8c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -311,6 +311,7 @@ class TestFormatSelection(unittest.TestCase):
         assert_syntax_error('bestvideo,,best')
         assert_syntax_error('+bestaudio')
         assert_syntax_error('bestvideo+')
+        assert_syntax_error('/')
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index efa3254ce..c608ff91a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -960,6 +960,8 @@ class YoutubeDL(object):
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
+                        if not current_selector:
+                            raise syntax_error('"/" must follow a format selector', start)
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])

From a346b1ff57a94382e80fd4edd5a6d4b91a7cb45e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Aug 2015 20:44:22 +0600
Subject: [PATCH 0960/2145] [bbc] Add support for vxp-playlist-data embeds
 (Closes #6453)

---
 youtube_dl/extractor/bbc.py | 45 ++++++++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 9a1b6e3dc..abc5a44a1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -526,6 +526,18 @@ class BBCIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video from video playlist embedded with vxp-playlist-data JSON
+        'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
+        'info_dict': {
+            'id': 'p02w6qjc',
+            'ext': 'mp4',
+            'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
+            'duration': 56,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         # single video story with digitalData
         'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
@@ -695,13 +707,36 @@ class BBCIE(BBCCoUkIE):
 
         if not medias:
             # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
-            media_asset_page = self._parse_json(
+            media_asset = self._search_regex(
+                r'mediaAssetPage\.init\(\s*({.+?}), "/',
+                webpage, 'media asset', default=None)
+            if media_asset:
+                media_asset_page = self._parse_json(media_asset, playlist_id, fatal=False)
+                medias = []
+                for video in media_asset_page.get('videos', {}).values():
+                    medias.extend(video.values())
+
+        if not medias:
+            # Multiple video playlist with single `now playing` entry (e.g.
+            # http://www.bbc.com/news/video_and_audio/must_see/33767813)
+            vxp_playlist = self._parse_json(
                 self._search_regex(
-                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                    r'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>',
+                    webpage, 'playlist data'),
                 playlist_id)
-            medias = []
-            for video in media_asset_page.get('videos', {}).values():
-                medias.extend(video.values())
+            playlist_medias = []
+            for item in vxp_playlist:
+                media = item.get('media')
+                if not media:
+                    continue
+                playlist_medias.append(media)
+                # Download single video if found media with asset id matching the video id from URL
+                if item.get('advert', {}).get('assetId') == playlist_id:
+                    medias = [media]
+                    break
+            # Fallback to the whole playlist
+            if not medias:
+                medias = playlist_medias
 
         entries = []
         for num, media_meta in enumerate(medias, start=1):

From 232541df441741d3d55605f03e28ec3c34249a5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 4 Aug 2015 22:29:23 +0200
Subject: [PATCH 0961/2145] [YoutubeDL] format spec: correctly handle dashes
 and other unused operators

'mp4-baseline-16x9' must be handled as a single string, but the '-' was treated as an operator.
---
 test/test_YoutubeDL.py  |  6 ++++++
 youtube_dl/YoutubeDL.py | 33 ++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 9a3c28f8c..0388c0bf3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -105,6 +105,7 @@ class TestFormatSelection(unittest.TestCase):
     def test_format_selection(self):
         formats = [
             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
             {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
             {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
             {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
@@ -136,6 +137,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], '35')
 
+        ydl = YDL({'format': 'example-with-dashes'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'example-with-dashes')
+
     def test_format_selection_audio(self):
         formats = [
             {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c608ff91a..1446b3254 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -933,6 +933,37 @@ class YoutubeDL(object):
                 else:
                     filter_parts.append(string)
 
+        def _remove_unused_ops(tokens):
+            # Remove operators that we don't use and join them with the sourrounding strings
+            # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
+            ALLOWED_OPS = ('/', '+', ',', '(', ')')
+            last_string, last_start, last_end, last_line = None, None, None, None
+            for type, string, start, end, line in tokens:
+                if type == tokenize.OP and string == '[':
+                    if last_string:
+                        yield tokenize.NAME, last_string, last_start, last_end, last_line
+                        last_string = None
+                    yield type, string, start, end, line
+                    # everything inside brackets will be handled by _parse_filter
+                    for type, string, start, end, line in tokens:
+                        yield type, string, start, end, line
+                        if type == tokenize.OP and string == ']':
+                            break
+                elif type == tokenize.OP and string in ALLOWED_OPS:
+                    if last_string:
+                        yield tokenize.NAME, last_string, last_start, last_end, last_line
+                        last_string = None
+                    yield type, string, start, end, line
+                elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
+                    if not last_string:
+                        last_string = string
+                        last_start = start
+                        last_end = end
+                    else:
+                        last_string += string
+            if last_string:
+                yield tokenize.NAME, last_string, last_start, last_end, last_line
+
         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
@@ -1111,7 +1142,7 @@ class YoutubeDL(object):
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
         try:
-            tokens = list(compat_tokenize_tokenize(stream.readline))
+            tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
         except tokenize.TokenError:
             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
 

From 8a5601e42f6974e6694f01089b4c7e014b6a1b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 19:52:04 +0600
Subject: [PATCH 0962/2145] [lynda] Fix login (Closes #6462)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index a00f6e5e5..39214de2f 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
+    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn\s*:\s*true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 

From 5b7dab2dd640c93ec0f63ca8b901e701679a4c7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:06:48 +0600
Subject: [PATCH 0963/2145] [lynda] Make login more robust

---
 youtube_dl/extractor/lynda.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 39214de2f..deead220a 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -17,7 +17,6 @@ from ..utils import (
 
 class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn\s*:\s*true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 
@@ -41,7 +40,7 @@ class LyndaBaseIE(InfoExtractor):
             request, None, 'Logging in as %s' % username)
 
         # Not (yet) logged in
-        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
+        m = re.search(r'loginResultJson\s*=\s*\'(?P<json>[^\']+)\';', login_page)
         if m is not None:
             response = m.group('json')
             response_json = json.loads(response)
@@ -70,7 +69,7 @@ class LyndaBaseIE(InfoExtractor):
                     request, None,
                     'Confirming log in and log out from another device')
 
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+        if all(not re.search(p, login_page) for p in ('isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
             raise ExtractorError('Unable to log in')
 
 

From 354b4b8604ec13ccf4bd89b9d1b77cb7246fe379 Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Wed, 5 Aug 2015 19:37:59 +0530
Subject: [PATCH 0964/2145] fixing xhamster file extraction

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b4ad513a0..9d025530f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -47,7 +47,7 @@ class XHamsterIE(InfoExtractor):
 
     def _real_extract(self, url):
         def extract_video_url(webpage):
-            mp4 = re.search(r'<video\s+.*?file="([^"]+)".*?>', webpage)
+            mp4 = re.search(r'file:\s+\'([^\']+)\'', webpage)
             if mp4 is None:
                 raise ExtractorError('Unable to extract media URL')
             else:

From be7a8379b47c35afe66abcc02aee597e5143b1d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:32:44 +0600
Subject: [PATCH 0965/2145] [xhamster] Make more robust

---
 youtube_dl/extractor/xhamster.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 9d025530f..481d79b89 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -46,12 +46,12 @@ class XHamsterIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        def extract_video_url(webpage):
-            mp4 = re.search(r'file:\s+\'([^\']+)\'', webpage)
-            if mp4 is None:
-                raise ExtractorError('Unable to extract media URL')
-            else:
-                return mp4.group(1)
+        def extract_video_url(webpage, name):
+            return self._search_regex(
+                [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
+                 r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
+                 r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
+                webpage, name, group='mp4')
 
         def is_hd(webpage):
             return '<div class=\'icon iconHD\'' in webpage
@@ -97,7 +97,9 @@ class XHamsterIE(InfoExtractor):
 
         hd = is_hd(webpage)
 
-        video_url = extract_video_url(webpage)
+        format_id = 'hd' if hd else 'sd'
+
+        video_url = extract_video_url(webpage, format_id)
         formats = [{
             'url': video_url,
             'format_id': 'hd' if hd else 'sd',
@@ -108,7 +110,7 @@ class XHamsterIE(InfoExtractor):
             mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
             webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
             if is_hd(webpage):
-                video_url = extract_video_url(webpage)
+                video_url = extract_video_url(webpage, 'hd')
                 formats.append({
                     'url': video_url,
                     'format_id': 'hd',

From 251a44b776264c17d7799e017b856143c6cacd9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:36:37 +0600
Subject: [PATCH 0966/2145] [xhamster] Fix thumbnail extraction

---
 youtube_dl/extractor/xhamster.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 481d79b89..b57e7c813 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -78,7 +78,10 @@ class XHamsterIE(InfoExtractor):
         uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
                                               webpage, 'uploader id', default='anonymous')
 
-        thumbnail = self._html_search_regex(r'<video\s+.*?poster="([^"]+)".*?>', webpage, 'thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
+             r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
+             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
                                                           webpage, 'duration', fatal=False))

From 3e4852247744b131600ba43275ab321eb1b32bb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:41:40 +0600
Subject: [PATCH 0967/2145] [xhamster] Fix uploader extraction

---
 youtube_dl/extractor/xhamster.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b57e7c813..06fedf840 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -22,7 +22,7 @@ class XHamsterIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'FemaleAgent Shy beauty takes the bait',
                 'upload_date': '20121014',
-                'uploader_id': 'Ruseful2011',
+                'uploader': 'Ruseful2011',
                 'duration': 893,
                 'age_limit': 18,
             }
@@ -34,7 +34,7 @@ class XHamsterIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Britney Spears  Sexy Booty',
                 'upload_date': '20130914',
-                'uploader_id': 'jojo747400',
+                'uploader': 'jojo747400',
                 'duration': 200,
                 'age_limit': 18,
             }
@@ -75,8 +75,9 @@ class XHamsterIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
-        uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
-                                              webpage, 'uploader id', default='anonymous')
+        uploader = self._html_search_regex(
+            r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",
+            webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
             [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
@@ -127,7 +128,7 @@ class XHamsterIE(InfoExtractor):
             'title': title,
             'description': description,
             'upload_date': upload_date,
-            'uploader_id': uploader_id,
+            'uploader': uploader,
             'thumbnail': thumbnail,
             'duration': duration,
             'view_count': view_count,

From 54a9328b205e8a2c916d59fd81bdb1ede25cf87a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 21:19:52 +0600
Subject: [PATCH 0968/2145] [generic] Expand jwplayer support

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8cef61c3c..6df89f814 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1655,7 +1655,7 @@ class GenericIE(InfoExtractor):
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
             found = filter_video(re.findall(
-                r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
+                r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
         if not found:
             # Flow player
             found = filter_video(re.findall(r'''(?xs)

From c71a3195afa8c2a9ed5fe0ffa56ff6c969147d91 Mon Sep 17 00:00:00 2001
From: Delon <liuxi326@qq.com>
Date: Wed, 5 Aug 2015 18:22:25 +0800
Subject: [PATCH 0969/2145] [tudou] Fix extracion

---
 youtube_dl/extractor/tudou.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c89de5ba4..9b934cb57 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -29,6 +29,8 @@ class TudouIE(InfoExtractor):
         }
     }]
 
+    _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
+	
     def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
@@ -76,6 +78,9 @@ class TudouIE(InfoExtractor):
                 'ext': ext,
                 'title': title,
                 'thumbnail': thumbnail_url,
+                'http_headers': {
+                    'Referer': self._PLAYER_URL,
+                },
             }
             result.append(part_info)
 

From 238755752f4f9169a1edda91067c8627afe19cce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:07:52 +0600
Subject: [PATCH 0970/2145] [tudou] Extract player URL from the webpage

---
 youtube_dl/extractor/tudou.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 9b934cb57..84fe71aef 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -30,7 +30,7 @@ class TudouIE(InfoExtractor):
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
-	
+
     def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
@@ -56,6 +56,10 @@ class TudouIE(InfoExtractor):
         thumbnail_url = self._search_regex(
             r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False)
 
+        player_url = self._search_regex(
+            r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
+            webpage, 'player URL', default=self._PLAYER_URL)
+
         segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
         segments = json.loads(segs_json)
         # It looks like the keys are the arguments that have to be passed as
@@ -79,7 +83,7 @@ class TudouIE(InfoExtractor):
                 'title': title,
                 'thumbnail': thumbnail_url,
                 'http_headers': {
-                    'Referer': self._PLAYER_URL,
+                    'Referer': player_url,
                 },
             }
             result.append(part_info)

From f535ec8278c8f465b47919d3f451571ae8ccfc7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:08:26 +0600
Subject: [PATCH 0971/2145] [xhamster] Remove unused import

---
 youtube_dl/extractor/xhamster.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 06fedf840..f76ee8fd4 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,7 +4,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     unified_strdate,
     str_to_int,
     int_or_none,

From c73cdd800f0dc7b465ac0b36d338875bb80c23aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:08:55 +0600
Subject: [PATCH 0972/2145] [xhamster] flake8

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f76ee8fd4..97315750f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -81,7 +81,7 @@ class XHamsterIE(InfoExtractor):
         thumbnail = self._search_regex(
             [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
              r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
-             webpage, 'thumbnail', fatal=False, group='thumbnail')
+            webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
                                                           webpage, 'duration', fatal=False))

From 51f267d9d4d26c3cd67f318a2040513946f2b4d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 22:01:01 +0600
Subject: [PATCH 0973/2145] [YoutubeDL:utils] Move percent encode non-ASCII
 URLs workaround to http_request and simplify (Closes #6457)

---
 youtube_dl/YoutubeDL.py | 21 ---------------------
 youtube_dl/utils.py     | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1446b3254..079d42ce8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1860,27 +1860,6 @@ class YoutubeDL(object):
 
     def urlopen(self, req):
         """ Start an HTTP download """
-
-        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
-        # always respected by websites, some tend to give out URLs with non percent-encoded
-        # non-ASCII characters (see telemb.py, ard.py [#3412])
-        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
-        # To work around aforementioned issue we will replace request's original URL with
-        # percent-encoded one
-        req_is_string = isinstance(req, compat_basestring)
-        url = req if req_is_string else req.get_full_url()
-        url_escaped = escape_url(url)
-
-        # Substitute URL if any change after escaping
-        if url != url_escaped:
-            if req_is_string:
-                req = url_escaped
-            else:
-                req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
-                req = req_type(
-                    url_escaped, data=req.data, headers=req.headers,
-                    origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
-
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 78dc2b449..c7db75f80 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -651,6 +651,26 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         return ret
 
     def http_request(self, req):
+        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
+        # always respected by websites, some tend to give out URLs with non percent-encoded
+        # non-ASCII characters (see telemb.py, ard.py [#3412])
+        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
+        # To work around aforementioned issue we will replace request's original URL with
+        # percent-encoded one
+        # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
+        # the code of this workaround has been moved here from YoutubeDL.urlopen()
+        url = req.get_full_url()
+        url_escaped = escape_url(url)
+
+        # Substitute URL if any change after escaping
+        if url != url_escaped:
+            req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+            new_req = req_type(
+                url_escaped, data=req.data, headers=req.headers,
+                origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+            new_req.timeout = req.timeout
+            req = new_req
+
         for h, v in std_headers.items():
             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
             # The dict keys are capitalized because of this bug by urllib

From bd690a9f9368095f561184778fb2f3ef12c66342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 22:01:31 +0600
Subject: [PATCH 0974/2145] [southpark:de] Add test for non-ASCII in URLs

---
 youtube_dl/extractor/southpark.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 7fb165a87..87b650468 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -45,6 +45,14 @@ class SouthParkDeIE(SouthParkIE):
             'title': 'The Government Won\'t Respect My Privacy',
             'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
         },
+    }, {
+        # non-ASCII characters in initial URL
+        'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
+        'playlist_count': 4,
+    }, {
+        # non-ASCII characters in redirect URL
+        'url': 'http://www.southpark.de/alle-episoden/s18e09',
+        'playlist_count': 4,
     }]
 
 

From 4f34cdb0a87a506d25a352ff265678c86cb9b979 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 23:56:44 +0600
Subject: [PATCH 0975/2145] [southpark:de] Skip test

---
 youtube_dl/extractor/southpark.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 87b650468..ad63a8785 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -53,6 +53,7 @@ class SouthParkDeIE(SouthParkIE):
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
         'playlist_count': 4,
+        'skip': 'Broken python 3',
     }]
 
 

From 671302b5c0ff8cefa5f26e599423ef7799b19631 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 00:08:11 +0600
Subject: [PATCH 0976/2145] [YoutubeDL] Remove unused imports

---
 youtube_dl/YoutubeDL.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 079d42ce8..cad6b026e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,7 +28,6 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
-    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_get_terminal_size,
@@ -40,7 +39,6 @@ from .compat import (
     compat_urllib_request,
 )
 from .utils import (
-    escape_url,
     ContentTooShortError,
     date_from_str,
     DateRange,
@@ -51,7 +49,6 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
-    HEADRequest,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,

From cd6b555e19c601d575679dd29da0080eda7f8890 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 6 Aug 2015 19:17:50 +0100
Subject: [PATCH 0977/2145] [dcn] add origin to api request and fix the test
 and check with flake8

---
 youtube_dl/extractor/dcn.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index f76ebda9e..d44e8cef0 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -1,4 +1,9 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
+
 
 class DcnIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
@@ -9,24 +14,29 @@ class DcnIE(InfoExtractor):
             'id': '17375',
             'ext': 'm3u8',
             'title': 'رحلة العمر : الحلقة 1',
-            'description': '"في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة1"',
+            'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
             'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
             'duration': '2041'
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json_data = self._download_json(
-            'http://admin.mangomolo.com/analytics/index.php/plus/video?id='+video_id,
-            video_id
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=' + video_id,
+            headers={'Origin': 'http://www.dcndigital.ae'}
         )
-        title = json_data['title_ar'];
-        thumbnail = 'http://admin.mangomolo.com/analytics/'+json_data['img'];
-        duration = json_data['duration'];
-        description = json_data['description_ar'];
+        json_data = self._download_json(request, video_id)
+        title = json_data['title_ar']
+        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data['img']
+        duration = json_data['duration']
+        description = json_data['description_ar']
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id='+json_data['id']+'&user_id='+json_data['user_id']+'&countries=Q0M=&w=100%&h=100%&filter=DENY&signature='+json_data['signature'],
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
             video_id
         )
         m3u8_url = self._html_search_regex(

From 3be3c622dc1d3d7b92c5268a079d202a9f2b0a5a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 6 Aug 2015 19:37:45 +0100
Subject: [PATCH 0978/2145] [shahid] generic errors handling and check with
 flake8

---
 youtube_dl/extractor/shahid.py | 42 ++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b3b45da24..57c159833 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
@@ -5,6 +8,7 @@ from ..utils import (
     int_or_none
 )
 
+
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
     _TESTS = [
@@ -23,7 +27,7 @@ class ShahidIE(InfoExtractor):
             }
         },
         {
-            #shahid plus subscriber only
+            # shahid plus subscriber only
             'url': 'https://shahid.mbc.net/ar/series/90497/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011.html',
             'only_matching': True
         }
@@ -32,31 +36,15 @@ class ShahidIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         player_info = ''
-        for line in self._search_regex( 'var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
+        for line in self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
             if '+' not in line and '(' not in line and ')' not in line:
                 player_info += line
         player_info = self._parse_json(js_to_json(player_info), video_id)
         video_id = player_info['id']
         player_type = player_info['playerType']
 
-        video_info = self._download_json(
-            player_info['url'] + '/' + player_type + '/' + video_id + 
-                '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
-            video_id
-        )['data']
-        if video_info['error']:
-            for error in video_info['error']:
-                raise ExtractorError(error)
-        video_info = video_info[player_type]
-        if video_info.get('availabilities').get('plus'):
-            raise ExtractorError('plus members only')
-        title = video_info['title']
-        thumbnail = video_info.get('thumbnailUrl')
-        categories = [category['name'] for category in video_info.get('genres')]
-        description = video_info.get('description')
-        duration = int_or_none(video_info.get('duration'))
-
         player_json_data = self._download_json(
             'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + player_info['type'] + '.html',
             video_id
@@ -66,8 +54,22 @@ class ShahidIE(InfoExtractor):
         else:
             for error in player_json_data['error'].values():
                 raise ExtractorError(error)
-            return
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
+
+        video_info = self._download_json(
+            player_info['url'] + '/' + player_type + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
+            video_id
+        )['data']
+        if video_info.get('error'):
+            for error in video_info['error']:
+                raise ExtractorError(error)
+        video_info = video_info[player_type]
+        title = video_info['title']
+        thumbnail = video_info.get('thumbnailUrl')
+        categories = [category['name'] for category in video_info.get('genres')]
+        description = video_info.get('description')
+        duration = int_or_none(video_info.get('duration'))
+
         return {
             'id': video_id,
             'title': title,

From 5a4d9ddb218e761fe7ab15d197690e0cb132a536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 01:26:40 +0600
Subject: [PATCH 0979/2145] [utils] Percent-encode redirect URL of Location
 header (Closes #6457)

---
 youtube_dl/utils.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c7db75f80..e265c7574 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -715,6 +715,17 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
+        if 300 <= resp.code < 400:
+            location = resp.headers.get('Location')
+            if location:
+                # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
+                if sys.version_info >= (3, 0):
+                    location = location.encode('iso-8859-1').decode('utf-8')
+                location_escaped = escape_url(location)
+                if location != location_escaped:
+                    del resp.headers['Location']
+                    resp.headers['Location'] = location_escaped
         return resp
 
     https_request = http_request

From 9663bd3abb78911bddad75742bd41006677d628e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 01:27:07 +0600
Subject: [PATCH 0980/2145] [southpark:de] Enable non-ASCII redirect URL test

---
 youtube_dl/extractor/southpark.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index ad63a8785..87b650468 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -53,7 +53,6 @@ class SouthParkDeIE(SouthParkIE):
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
         'playlist_count': 4,
-        'skip': 'Broken python 3',
     }]
 
 

From 3eb5fdb58112032a9831eda1d2e3b8a151ea217f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 6 Aug 2015 22:55:43 +0200
Subject: [PATCH 0981/2145] release 2015.08.06

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fa157cadb..b81d5e658 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.28'
+__version__ = '2015.08.06'

From 430b092a5f59fbe407b92ebcb0c42b9f7062a334 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 6 Aug 2015 23:06:21 +0200
Subject: [PATCH 0982/2145] release 2015.08.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b81d5e658..9f209499c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.06'
+__version__ = '2015.08.06.1'

From 6d30cf04db9c9662dbb30c2490e24eb5c6dca4c3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 7 Aug 2015 10:01:18 +0100
Subject: [PATCH 0983/2145] [dcn] fix type and key errors

---
 youtube_dl/extractor/dcn.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index d44e8cef0..22ff35b56 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
+from ..utils import int_or_none
 
 
 class DcnIE(InfoExtractor):
@@ -16,7 +17,7 @@ class DcnIE(InfoExtractor):
             'title': 'رحلة العمر : الحلقة 1',
             'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
             'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
-            'duration': '2041'
+            'duration': 2041
         },
         'params': {
             # m3u8 download
@@ -32,9 +33,9 @@ class DcnIE(InfoExtractor):
         )
         json_data = self._download_json(request, video_id)
         title = json_data['title_ar']
-        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data['img']
-        duration = json_data['duration']
-        description = json_data['description_ar']
+        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data.get('img')
+        duration = int_or_none(json_data.get('duration'))
+        description = json_data.get('description_ar')
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
             video_id

From 8002ac9e0a88d918735c06599dbf8f2005f79666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:04:44 +0600
Subject: [PATCH 0984/2145] [nowtv] Add support for .at TLD

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index ad938fb62..78e8851c0 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl

From acc1adbe7ab93657cd4d303cee1fba4464931a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:50:54 +0600
Subject: [PATCH 0985/2145] [nowtv] Add support for .ch TLD

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 78e8851c0..fc21d8e3f 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl

From 0f422256d6eea5aff062a4c35d7434cd118c7a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:51:09 +0600
Subject: [PATCH 0986/2145] [nowtv] Add .at test

---
 youtube_dl/extractor/nowtv.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index fc21d8e3f..66c627bec 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -127,6 +127,9 @@ class NowTVIE(InfoExtractor):
     }, {
         'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f94639fadf91312bf3365802981f506ecba698dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 00:06:03 +0600
Subject: [PATCH 0987/2145] [dcn] Improve

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/dcn.py      | 78 ++++++++++++++++++++++----------
 2 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eb8ef1fe3..922d9b3d8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -118,7 +118,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
-from .dcn import DcnIE
+from .dcn import DCNIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 22ff35b56..b98a6c032 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,22 +2,30 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import int_or_none
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
-class DcnIE(InfoExtractor):
+class DCNIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
         'info_dict':
         {
             'id': '17375',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'رحلة العمر : الحلقة 1',
-            'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
-            'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
-            'duration': 2041
+            'description': 'md5:0156e935d870acb8ef0a66d24070c6d6',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2041,
+            'timestamp': 1227504126,
+            'upload_date': '20081124',
         },
         'params': {
             # m3u8 download
@@ -27,30 +35,50 @@ class DcnIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         request = compat_urllib_request.Request(
-            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=' + video_id,
-            headers={'Origin': 'http://www.dcndigital.ae'}
-        )
-        json_data = self._download_json(request, video_id)
-        title = json_data['title_ar']
-        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data.get('img')
-        duration = int_or_none(json_data.get('duration'))
-        description = json_data.get('description_ar')
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
+            headers={'Origin': 'http://www.dcndigital.ae'})
+
+        video = self._download_json(request, video_id)
+        title = video.get('title_en') or video['title_ar']
+
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
-            video_id
-        )
-        m3u8_url = self._html_search_regex(
-            r'file:\s*"([^"]+)',
-            webpage,
-            'm3u8_url'
-        )
-        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
+            + compat_urllib_parse.urlencode({
+                'id': video['id'],
+                'user_id': video['user_id'],
+                'signature': video['signature'],
+                'countries': 'Q0M=',
+                'filter': 'DENY',
+            }), video_id)
+
+        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+
+        img = video.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video.get('duration'))
+        description = video.get('description_en') or video.get('description_ar')
+        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
+
         return {
             'id': video_id,
             'title': title,
+            'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'description': description,
+            'timestamp': timestamp,
             'formats': formats,
         }

From 4a7434d0b09e14b773c2d278c8299efa6225b84e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 00:19:40 +0600
Subject: [PATCH 0988/2145] [dcn] Simplify _VALID_URL

---
 youtube_dl/extractor/dcn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index b98a6c032..82261e25c 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class DCNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
         'info_dict':

From fd5d8270dcd6d8baada3390a4a1cae5bdbcb6da4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 01:10:41 +0600
Subject: [PATCH 0989/2145] [clipfish] Fix extraction, minimize requests, get
 rid of drm hds, extract m3u8 and more metadata

---
 youtube_dl/extractor/clipfish.py | 56 ++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 09dfaac60..7af903571 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,18 +1,19 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     int_or_none,
     js_to_json,
-    determine_ext,
+    parse_iso8601,
+    remove_end,
 )
 
 
 class ClipfishIE(InfoExtractor):
-    IE_NAME = 'clipfish'
-
-    _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
         'md5': '79bc922f3e8a9097b3d68a93780fd475',
@@ -20,35 +21,48 @@ class ClipfishIE(InfoExtractor):
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
+            'timestamp': 1370938118,
+            'upload_date': '20130611',
             'duration': 82,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_info = self._parse_json(
-            js_to_json(self._html_search_regex('var videoObject = ({[^}]+?})', webpage, 'videoObject')),
-            video_id
-        )
-        info_url = self._parse_json(
-            js_to_json(self._html_search_regex('var globalFlashvars = ({[^}]+?})', webpage, 'globalFlashvars')),
-            video_id
-        )['data']
 
-        doc = self._download_xml(
-            info_url, video_id, note='Downloading info page')
-        title = doc.find('title').text
-        video_url = doc.find('filename').text
-        thumbnail = doc.find('imageurl').text
-        duration = int_or_none(video_info['length'])
-        formats = [{'url': video_info['videourl']},{'url': video_url}]
+        webpage = self._download_webpage(url, video_id)
+
+        video_info = self._parse_json(
+            js_to_json(self._html_search_regex(
+                '(?s)videoObject\s*=\s*({.+?});', webpage, 'video object')),
+            video_id)
+
+        formats = []
+        for video_url in re.findall(r'var\s+videourl\s*=\s*"([^"]+)"', webpage):
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.append({
+                    'url': video_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+                    'ext': 'mp4',
+                    'format_id': 'hls',
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': ext,
+                })
         self._sort_formats(formats)
 
+        title = remove_end(self._og_search_title(webpage), ' - Video')
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(video_info.get('length'))
+        timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage, 'upload date'))
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'timestamp': timestamp,
         }

From 8a37aa1517ccc474b3e2831b77e48534cb8ed47c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 01:55:59 +0600
Subject: [PATCH 0990/2145] [extractor/generic] Expand ooyala regex (Closes
 #6485)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6df89f814..649c0bce6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1320,7 +1320,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for Ooyala videos
-        mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
+        mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))

From bf94d763ba73e09fd77d25110c7219254b63c786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 02:00:49 +0600
Subject: [PATCH 0991/2145] [extractor/generic] Add test for #6485

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 649c0bce6..469909a51 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -236,6 +236,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
+        {
+            # ooyala video embedded with http://player.ooyala.com/iframe.js
+            'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
+            'info_dict': {
+                'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
+                'ext': 'mp4',
+                'title': '"Steve Jobs: Man in the Machine" trailer',
+                'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # multiple ooyala embeds on SBN network websites
         {
             'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',

From c29458f3ec77072e9c17169b78871bf4473134d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 7 Aug 2015 21:38:50 +0100
Subject: [PATCH 0992/2145] [shahid] change the tests

---
 youtube_dl/extractor/shahid.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 57c159833..b2050525e 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -13,13 +13,13 @@ class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
     _TESTS = [
         {
-            'url': 'https://shahid.mbc.net/ar/episode/108084/%D8%AE%D9%88%D8%A7%D8%B7%D8%B1-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-11-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+            'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
             'info_dict': {
-                'id': '108084',
+                'id': '90574',
                 'ext': 'm3u8',
-                'title': 'خواطر الموسم 11 الحلقة 1',
-                'description': 'بسم الله',
-                'duration': 1166,
+                'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
+                'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
+                'duration': 2972,
             },
             'params': {
                 # m3u8 download
@@ -28,7 +28,7 @@ class ShahidIE(InfoExtractor):
         },
         {
             # shahid plus subscriber only
-            'url': 'https://shahid.mbc.net/ar/series/90497/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011.html',
+            'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
             'only_matching': True
         }
     ]

From e0ac521438218e978b9c4bbcd92cfc2d5fef79cb Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Thu, 6 Aug 2015 22:42:58 +0530
Subject: [PATCH 0993/2145] adding support for axel download manager

---
 youtube_dl/downloader/external.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1d5cc9904..30699934b 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -83,6 +83,16 @@ class CurlFD(ExternalFD):
         return cmd
 
 
+class AxelFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        for key, val in info_dict['http_headers'].items():
+            cmd += ['-H', '%s: %s' % (key, val)]
+        cmd += self._configuration_args()
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']

From 5b0c40da24b5ddb789428de731e02ac8759a363c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 03:36:29 +0600
Subject: [PATCH 0994/2145] [extractor/common] Expand meta regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..507ea5ec0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -636,7 +636,7 @@ class InfoExtractor(object):
     @staticmethod
     def _meta_regex(prop):
         return r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    (?=[^>]+(?:itemprop|name|property|id)=(["\']?)%s\1)
                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 
     def _og_search_property(self, prop, html, name=None, **kargs):

From 3550821fb4ca2f0e47542a7fa16b6543b06df724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 03:38:55 +0600
Subject: [PATCH 0995/2145] [periscope] Add extractor (Closes #5850, closes
 #6459)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/periscope.py | 66 +++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 youtube_dl/extractor/periscope.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 922d9b3d8..bd86a5be2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,6 +432,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .periscope import PeriscopeIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
new file mode 100644
index 000000000..5219e1a75
--- /dev/null
+++ b/youtube_dl/extractor/periscope.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    unescapeHTML,
+)
+
+
+class PeriscopeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
+        'md5': '65b57957972e503fcbbaeed8f4fa04ca',
+        'info_dict': {
+            'id': '56102209',
+            'ext': 'mp4',
+            'title': 'Bec Boop - 🚠✈️🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈️🚠 #BoopScope 🎀💗',
+            'timestamp': 1438978559,
+            'upload_date': '20150807',
+            'uploader': 'Bec Boop',
+            'uploader_id': '1465763',
+        },
+        'skip': 'Expires in 24 hours',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        replay = self._download_json(
+            'https://api.periscope.tv/api/v2/getAccessPublic?token=%s' % video_id, video_id)
+
+        video_url = replay['replay_url']
+
+        webpage = self._download_webpage(url, video_id)
+
+        broadcast_data = self._parse_json(
+            unescapeHTML(self._html_search_meta(
+                'broadcast-data', webpage, 'broadcast data', fatal=True)),
+            video_id)
+
+        broadcast = broadcast_data['broadcast']
+        status = broadcast['status']
+
+        uploader = broadcast.get('user_display_name') or broadcast_data.get('user', {}).get('display_name')
+        uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
+
+        title = '%s - %s' % (uploader, status) if uploader else status
+        timestamp = parse_iso8601(broadcast.get('created_at'))
+
+        thumbnails = [{
+            'url': broadcast[image],
+        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
+
+        return {
+            'id': broadcast.get('id') or video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            'protocol': 'm3u8_native',
+            'title': title,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'thumbnails': thumbnails,
+        }

From 621d6a9516e0f9cd8c45e12904f4d4b7615e7fb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 04:00:52 +0600
Subject: [PATCH 0996/2145] [periscope] Switch to API for broadcast data

---
 youtube_dl/extractor/periscope.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 5219e1a75..11648a511 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -25,21 +25,17 @@ class PeriscopeIE(InfoExtractor):
         'skip': 'Expires in 24 hours',
     }
 
+    def _call_api(self, method, token):
+        return self._download_json(
+            'https://api.periscope.tv/api/v2/%s?token=%s' % (method, token), token)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        replay = self._download_json(
-            'https://api.periscope.tv/api/v2/getAccessPublic?token=%s' % video_id, video_id)
+        token = self._match_id(url)
 
+        replay = self._call_api('getAccessPublic', token)
         video_url = replay['replay_url']
 
-        webpage = self._download_webpage(url, video_id)
-
-        broadcast_data = self._parse_json(
-            unescapeHTML(self._html_search_meta(
-                'broadcast-data', webpage, 'broadcast data', fatal=True)),
-            video_id)
-
+        broadcast_data = self._call_api('getBroadcastPublic', token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
 
@@ -54,7 +50,7 @@ class PeriscopeIE(InfoExtractor):
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
         return {
-            'id': broadcast.get('id') or video_id,
+            'id': broadcast.get('id') or token,
             'url': video_url,
             'ext': 'mp4',
             'protocol': 'm3u8_native',

From 1e83741c9a5d67e8bbe65510d41b558361496fe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:33:53 +0600
Subject: [PATCH 0997/2145] [periscope] Add support for running streams

---
 youtube_dl/extractor/periscope.py | 34 +++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 11648a511..de53b752d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,13 +2,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    unescapeHTML,
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
 )
+from ..utils import parse_iso8601
 
 
 class PeriscopeIE(InfoExtractor):
+    IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
     _TEST = {
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -32,9 +34,6 @@ class PeriscopeIE(InfoExtractor):
     def _real_extract(self, url):
         token = self._match_id(url)
 
-        replay = self._call_api('getAccessPublic', token)
-        video_url = replay['replay_url']
-
         broadcast_data = self._call_api('getBroadcastPublic', token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
@@ -43,20 +42,37 @@ class PeriscopeIE(InfoExtractor):
         uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
 
         title = '%s - %s' % (uploader, status) if uploader else status
+        state = broadcast.get('state').lower()
+        if state == 'running':
+            title = self._live_title(title)
         timestamp = parse_iso8601(broadcast.get('created_at'))
 
         thumbnails = [{
             'url': broadcast[image],
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
+        stream = self._call_api('getAccessPublic', token)
+
+        formats = []
+        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
+            video_url = stream.get(format_id + '_url')
+            if not video_url:
+                continue
+            f = {
+                'url': video_url,
+                'ext': 'flv' if format_id == 'rtmp' else 'mp4',
+            }
+            if format_id != 'rtmp':
+                f['protocol'] = 'm3u8_native' if state == 'ended' else 'm3u8'
+            formats.append(f)
+        self._sort_formats(formats)
+
         return {
             'id': broadcast.get('id') or token,
-            'url': video_url,
-            'ext': 'mp4',
-            'protocol': 'm3u8_native',
             'title': title,
             'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'thumbnails': thumbnails,
+            'formats': formats,
         }

From 428e4e4a850df81031e8267dddf759da605639e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:37:38 +0600
Subject: [PATCH 0998/2145] [quickscope] Add extractor

---
 youtube_dl/extractor/__init__.py  |  5 ++++-
 youtube_dl/extractor/periscope.py | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd86a5be2..e38e77a27 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,7 +432,10 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .periscope import PeriscopeIE
+from .periscope import (
+    PeriscopeIE,
+    QuickscopeIE,
+)
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index de53b752d..578b53a24 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -76,3 +76,24 @@ class PeriscopeIE(InfoExtractor):
             'thumbnails': thumbnails,
             'formats': formats,
         }
+
+
+class QuickscopeIE(InfoExtractor):
+    IE_DESC = 'Quisck Scope'
+    _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://watchonperiscope.com/broadcast/56180087',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        broadcast_id = self._match_id(url)
+        request = compat_urllib_request.Request(
+            'https://watchonperiscope.com/api/accessChannel', compat_urllib_parse.urlencode({
+                'broadcast_id': broadcast_id,
+                'entry_ticket': '',
+                'from_push': 'false',
+                'uses_sessions': 'true',
+            }).encode('utf-8'))
+        return self.url_result(
+            self._download_json(request, broadcast_id)['share_url'], 'Periscope')

From b2f82948ee5eadc483c01dc589b82426bb32ba68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:40:41 +0600
Subject: [PATCH 0999/2145] [quickscope] Fix typo

---
 youtube_dl/extractor/periscope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 578b53a24..8ad936758 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -79,7 +79,7 @@ class PeriscopeIE(InfoExtractor):
 
 
 class QuickscopeIE(InfoExtractor):
-    IE_DESC = 'Quisck Scope'
+    IE_DESC = 'Quick Scope'
     _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
     _TEST = {
         'url': 'https://watchonperiscope.com/broadcast/56180087',

From 59e89e62d7b45554cef502dc4986f35618110679 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 8 Aug 2015 12:59:10 +0100
Subject: [PATCH 1000/2145] [shahid] add default fallbacks for extracting api
 vars

---
 youtube_dl/extractor/shahid.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b2050525e..399140189 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -33,20 +33,30 @@ class ShahidIE(InfoExtractor):
         }
     ]
 
+    _api_vars = {
+        'type': 'player',
+        'url': 'http://api.shahid.net/api/v1_1',
+        'playerType': 'episode',
+    }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         player_info = ''
-        for line in self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
-            if '+' not in line and '(' not in line and ')' not in line:
-                player_info += line
-        player_info = self._parse_json(js_to_json(player_info), video_id)
-        video_id = player_info['id']
-        player_type = player_info['playerType']
+        flash_vars = self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars', None)
+        if flash_vars is not None:
+            for line in flash_vars.splitlines():
+                if '+' not in line and '(' not in line and ')' not in line:
+                    player_info += line
+            player_info = self._parse_json(player_info, video_id, js_to_json, False)
+            if player_info is not None:
+                for key in self._api_vars:
+                    if key in player_info:
+                        self._api_vars[key] = player_info[key]
 
         player_json_data = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + player_info['type'] + '.html',
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + self._api_vars['type'] + '.html',
             video_id
         )['data']
         if 'url' in player_json_data:
@@ -57,13 +67,13 @@ class ShahidIE(InfoExtractor):
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
 
         video_info = self._download_json(
-            player_info['url'] + '/' + player_type + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
+            self._api_vars['url'] + '/' + self._api_vars['playerType'] + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
             video_id
         )['data']
         if video_info.get('error'):
             for error in video_info['error']:
                 raise ExtractorError(error)
-        video_info = video_info[player_type]
+        video_info = video_info[self._api_vars['playerType']]
         title = video_info['title']
         thumbnail = video_info.get('thumbnailUrl')
         categories = [category['name'] for category in video_info.get('genres')]

From 154655a85ae8b7740aa9fe7821544050fd65641b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 19:21:05 +0600
Subject: [PATCH 1001/2145] [downloader/external] Respect
 --no-check-certificate for wget

---
 youtube_dl/downloader/external.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 30699934b..07ce59f7d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,9 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _no_check_certificate(self, command_option):
+        return [command_option] if self.params.get('nocheckcertificate', False) else []
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -99,6 +102,7 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._no_check_certificate('--no-check-certificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From b465083f45e63fe8aeb0255b5cea7dfbf0770a2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:27:10 +0600
Subject: [PATCH 1002/2145] [sexykarma] Fix test

---
 youtube_dl/extractor/sexykarma.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index 6446d26dc..e33483674 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -29,6 +29,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',

From b61b7787cbef408154695bbb9f5c3d29a70fdd38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:30:57 +0600
Subject: [PATCH 1003/2145] [91porn] Extract age limit

---
 youtube_dl/extractor/porn91.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 72d1b2718..3e15533e9 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -22,6 +22,7 @@ class Porn91IE(InfoExtractor):
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
             'ext': 'mp4',
             'duration': 431,
+            'age_limit': 18,
         }
     }
 
@@ -68,4 +69,5 @@ class Porn91IE(InfoExtractor):
             'url': video_url,
             'duration': duration,
             'comment_count': comment_count,
+            'age_limit': self._rta_search(webpage),
         }

From 8e2b1be12791b4e62c463562b570661e7b2c5852 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:42:50 +0600
Subject: [PATCH 1004/2145] [test/helper] Make age_limit checkable field

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index c8b34654d..cb6eec8d9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -160,7 +160,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                           for key, value in got_dict.items()
-                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location', 'age_limit'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):

From 18c3281f9e1e32e00c778b149137fc91accb3b1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:43:20 +0600
Subject: [PATCH 1005/2145] [24video] Fix test

---
 youtube_dl/extractor/fourtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index b2284ab01..3bb4f6239 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -32,6 +32,7 @@ class FourTubeIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }
 

From 464e792496665b2e3dcabf5df43a45604673730a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:51:21 +0600
Subject: [PATCH 1006/2145] [vpro] Override npo IE_NAME

---
 youtube_dl/extractor/npo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 0c2d02c10..eb12fb810 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -407,6 +407,7 @@ class NPORadioFragmentIE(InfoExtractor):
 
 
 class VPROIE(NPOIE):
+    IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
 
     _TESTS = [

From d7bb8884afc8651b0ad86046dcd56a5330c98dd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:58:24 +0600
Subject: [PATCH 1007/2145] [break] Add age_limit to test

---
 youtube_dl/extractor/breakcom.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 809287d14..aa08051b1 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -18,6 +18,7 @@ class BreakIE(InfoExtractor):
             'id': '2468056',
             'ext': 'mp4',
             'title': 'When Girls Act Like D-Bags',
+            'age_limit': 13,
         }
     }, {
         'url': 'http://www.break.com/video/ugc/baby-flex-2773063',

From 9f2e7c2f34c48942a2a3e55532dd0d0ef8ed4d98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:04:48 +0600
Subject: [PATCH 1008/2145] [ok] Add age_limit to tests

---
 youtube_dl/extractor/odnoklassniki.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 215ffe87b..e5fd1ba04 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -25,6 +25,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
+            'age_limit': 0,
         },
     }, {
         # metadataUrl
@@ -38,6 +39,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': '534380003155',
             'uploader': 'Андрей Мещанинов',
             'like_count': int,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',

From 887e9bc7b561f9b2b97dec8f99f9c04392d95d40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:08:54 +0600
Subject: [PATCH 1009/2145] [ok] Update tests

---
 youtube_dl/extractor/odnoklassniki.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index e5fd1ba04..003d27de7 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -16,12 +16,13 @@ class OdnoklassnikiIE(InfoExtractor):
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
-        'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
+        'md5': '6ba728d85d60aa2e6dd37c9e70fdc6bc',
         'info_dict': {
             'id': '20079905452',
             'ext': 'mp4',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'duration': 100,
+            'upload_date': '20141207',
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
@@ -36,8 +37,9 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Девушка без комплексов ...',
             'duration': 191,
+            'upload_date': '20150518',
             'uploader_id': '534380003155',
-            'uploader': 'Андрей Мещанинов',
+            'uploader': '☭ Андрей Мещанинов ☭',
             'like_count': int,
             'age_limit': 0,
         },

From c8d1be772daa496759bd85cb95c4ec799294c7f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:11:06 +0600
Subject: [PATCH 1010/2145] [rutube] Add age_limit to test

---
 youtube_dl/extractor/rutube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 5b1c3577a..d94dc7399 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -30,6 +30,7 @@ class RutubeIE(InfoExtractor):
             'uploader': 'NTDRussian',
             'uploader_id': '29790',
             'upload_date': '20131016',
+            'age_limit': 0,
         },
         'params': {
             # It requires ffmpeg (m3u8 download)

From 08df685fe7764ef9f7dc271075340e4effc5e621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 08:51:37 +0600
Subject: [PATCH 1011/2145] [videolectures] Fix _VALID_URL for
 test_no_duplicates to pass

---
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 24584dc80..ef2da5632 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)(?:/?[#?].*)?$'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
     _TEST = {

From 12bb392a0ff8adbde2ced75b0c4976d0aabc7f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 17:10:40 +0600
Subject: [PATCH 1012/2145] [vimeo] Fix password protected videos (Closes
 #6507)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 10d6745af..4c4e3c72a 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -203,7 +203,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'xsrft=%s' % token)
+        password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')

From 8d6765cf48138cc44fdbaee4e8c7a199ae348bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:07:18 +0600
Subject: [PATCH 1013/2145] [extractor/generic] Add generic support for xspf
 playist extraction

---
 youtube_dl/extractor/common.py | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index def6caa0d..e201ea6db 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -39,6 +39,8 @@ from ..utils import (
     sanitize_filename,
     unescapeHTML,
     url_basename,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
@@ -1142,6 +1144,45 @@ class InfoExtractor(object):
             })
         return subtitles
 
+    def _extract_xspf_playlist(self, playlist_url, playlist_id):
+        playlist = self._download_xml(
+            playlist_url, playlist_id, 'Downloading xpsf playlist',
+            'Unable to download xspf manifest')
+
+        NS_MAP = {
+            'xspf': 'http://xspf.org/ns/0/',
+            's1': 'http://static.streamone.nl/player/ns/0',
+        }
+
+        entries = []
+        for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
+            title = xpath_text(
+                track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+            description = xpath_text(
+                track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+            thumbnail = xpath_text(
+                track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+            duration = float_or_none(
+                xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
+
+            formats = [{
+                'url': location.text,
+                'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+                'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+                'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+            } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
+            self._sort_formats(formats)
+
+            entries.append({
+                'id': playlist_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+            })
+        return entries
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From e0b9d78fab76e2c2819c8a9a7512ad4533319b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:09:50 +0600
Subject: [PATCH 1014/2145] [extractor/common] Clarify playlists can have
 description field

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e201ea6db..9b4775e0a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -204,8 +204,8 @@ class InfoExtractor(object):
     There must be a key "entries", which is a list, an iterable, or a PagedList
     object, each element of which is a valid dictionary by this specification.
 
-    Additionally, playlists can have "title" and "id" attributes with the same
-    semantics as videos (see above).
+    Additionally, playlists can have "title", "description" and "id" attributes
+    with the same semantics as videos (see above).
 
 
     _type "multi_video" indicates that there are multiple videos that

From 3a30508b943c044e5f684b703ff58ac352686f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:11:23 +0600
Subject: [PATCH 1015/2145] [telegraaf] Add extractor (Closes #6492)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/telegraaf.py | 35 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/telegraaf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e38e77a27..dad3ec87f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -596,6 +596,7 @@ from .techtalks import TechTalksIE
 from .ted import TEDIE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
+from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
new file mode 100644
index 000000000..6f8333cfc
--- /dev/null
+++ b/youtube_dl/extractor/telegraaf.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class TelegraafIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?telegraaf\.nl/tv/(?:[^/]+/)+(?P<id>\d+)/[^/]+\.html'
+    _TEST = {
+        'url': 'http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html',
+        'md5': '83245a9779bcc4a24454bfd53c65b6dc',
+        'info_dict': {
+            'id': '24353229',
+            'ext': 'mp4',
+            'title': 'Tikibad ontruimd wegens brand',
+            'description': 'md5:05ca046ff47b931f9b04855015e163a4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 33,
+        },
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist_url = self._search_regex(
+            r"iframe\.loadPlayer\('([^']+)'", webpage, 'player')
+
+        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
+        title = remove_end(self._og_search_title(webpage), ' - VIDEO')
+        description = self._og_search_description(webpage)
+
+        return self.playlist_result(entries, playlist_id, title, description)

From f32143469fd0a2720bd40908ea8360490983b97d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:15:00 +0600
Subject: [PATCH 1016/2145] [tweakers] Use _extract_xspf_playlist

---
 youtube_dl/extractor/tweakers.py | 42 +++-----------------------------
 1 file changed, 4 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index c80ec15cf..4bbe76e96 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,41 +25,7 @@ class TweakersIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        playlist = self._download_xml(
-            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
-            video_id)
-
-        NS_MAP = {
-            'xspf': 'http://xspf.org/ns/0/',
-            's1': 'http://static.streamone.nl/player/ns/0',
-        }
-
-        track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
-
-        title = xpath_text(
-            track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
-        description = xpath_text(
-            track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
-        thumbnail = xpath_text(
-            track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
-        duration = float_or_none(
-            xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
-            1000)
-
-        formats = [{
-            'url': location.text,
-            'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
-            'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
-            'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
-        } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        playlist_id = self._match_id(url)
+        entries = self._extract_xspf_playlist(
+            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % playlist_id, playlist_id)
+        return self.playlist_result(entries, playlist_id)

From 0dcb318f622d944ad0f5c23c32c9bc9b00e76aaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:15:20 +0600
Subject: [PATCH 1017/2145] [tweakers] Fix test

---
 youtube_dl/extractor/tweakers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 4bbe76e96..6eeffb1cc 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -13,7 +13,7 @@ class TweakersIE(InfoExtractor):
     _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': '1b5afa817403bb5baa08359dca31e6df',
+        'md5': '3147e4ddad366f97476a93863e4557c8',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',

From 98044462b1035000a44b35a41f4f780b2e844f2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:18:50 +0600
Subject: [PATCH 1018/2145] [extractor/common] Use playlist id as default title

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9b4775e0a..be91e03e9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1157,7 +1157,7 @@ class InfoExtractor(object):
         entries = []
         for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
             title = xpath_text(
-                track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+                track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
             description = xpath_text(
                 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
             thumbnail = xpath_text(

From fb2f339fec20c35cb62c1da682e0dfd418faef81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:21:25 +0600
Subject: [PATCH 1019/2145] [dhm] Use _extract_xspf_playlist

---
 youtube_dl/extractor/dhm.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 3ed1f1663..127eb0439 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -34,24 +34,14 @@ class DHMIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        playlist_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, playlist_id)
 
         playlist_url = self._search_regex(
             r"file\s*:\s*'([^']+)'", webpage, 'playlist url')
 
-        playlist = self._download_xml(playlist_url, video_id)
-
-        track = playlist.find(
-            './{http://xspf.org/ns/0/}trackList/{http://xspf.org/ns/0/}track')
-
-        video_url = xpath_text(
-            track, './{http://xspf.org/ns/0/}location',
-            'video url', fatal=True)
-        thumbnail = xpath_text(
-            track, './{http://xspf.org/ns/0/}image',
-            'thumbnail')
+        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
 
         title = self._search_regex(
             [r'dc:title="([^"]+)"', r'<title> &raquo;([^<]+)</title>'],
@@ -63,11 +53,10 @@ class DHMIE(InfoExtractor):
             r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
             webpage, 'duration', default=None))
 
-        return {
-            'id': video_id,
-            'url': video_url,
+        entries[0].update({
             'title': title,
             'description': description,
             'duration': duration,
-            'thumbnail': thumbnail,
-        }
+        })
+
+        return self.playlist_result(entries, playlist_id)

From 942acef594428b5f5c7e0ed7860cb6d725d8f1e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:41:55 +0600
Subject: [PATCH 1020/2145] [extractor/common] Extract _parse_xspf

---
 youtube_dl/extractor/common.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index be91e03e9..5982055be 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1144,11 +1144,15 @@ class InfoExtractor(object):
             })
         return subtitles
 
-    def _extract_xspf_playlist(self, playlist_url, playlist_id):
-        playlist = self._download_xml(
+    def _extract_xspf_playlist(self, playlist_url, playlist_id, fatal=True):
+        xspf = self._download_xml(
             playlist_url, playlist_id, 'Downloading xpsf playlist',
-            'Unable to download xspf manifest')
+            'Unable to download xspf manifest', fatal=fatal)
+        if xspf is False:
+            return []
+        return self._parse_xspf(xspf, playlist_id)
 
+    def _parse_xspf(self, playlist, playlist_id):
         NS_MAP = {
             'xspf': 'http://xspf.org/ns/0/',
             's1': 'http://static.streamone.nl/player/ns/0',

From 729accb48221bd72e40076939616792c1c6fc15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:43:42 +0600
Subject: [PATCH 1021/2145] [extractor/generic] Add support for xspf playlists

---
 youtube_dl/extractor/generic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 901f77304..a382d6be4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1198,6 +1198,8 @@ class GenericIE(InfoExtractor):
                 return self._extract_rss(url, video_id, doc)
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
                 return self._parse_smil(doc, url, video_id)
+            elif doc.tag == '{http://xspf.org/ns/0/}playlist':
+                return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
         except compat_xml_parse_error:
             pass
 
@@ -1799,7 +1801,8 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
-            if determine_ext(video_url) == 'smil':
+            ext = determine_ext(video_url)
+            if ext == 'smil':
                 entries.append({
                     'id': video_id,
                     'formats': self._extract_smil_formats(video_url, video_id),
@@ -1807,6 +1810,8 @@ class GenericIE(InfoExtractor):
                     'title': video_title,
                     'age_limit': age_limit,
                 })
+            elif ext == 'xspf':
+                return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             else:
                 entries.append({
                     'id': video_id,

From 1de5cd3ba51ce67d9a1cd3b40157058e78e46692 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:47:08 +0600
Subject: [PATCH 1022/2145] [extractor/generic] Add test for xspf playlist

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a382d6be4..4756a658f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -198,6 +198,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
+        {
+            'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
+            'info_dict': {
+                'id': 'mZlp2ctYIUEB',
+                'ext': 'mp4',
+                'title': 'Tikibad ontruimd wegens brand',
+                'description': 'md5:05ca046ff47b931f9b04855015e163a4',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 33,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 0791ac1b4415601f464f9656a4485b3ae6b67f4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:47:58 +0600
Subject: [PATCH 1023/2145] [extractor/generic] Clarify comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4756a658f..376feecae 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1206,7 +1206,7 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed or a SMIL file?
+        # Is it an RSS feed, a SMIL file or a XSPF playlist?
         try:
             doc = parse_xml(webpage)
             if doc.tag == 'rss':

From 27c7114af6b82bfe8be6b8e4dfa6e11dd1356044 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 9 Aug 2015 20:13:02 +0200
Subject: [PATCH 1024/2145] release 2015.08.09

---
 README.md              | 2 +-
 docs/supportedsites.md | 8 ++++++--
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 02b9775f9..15baf75ce 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,httpie,wget
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,axel,curl,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 657935dc6..e21471102 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -86,7 +86,7 @@
  - **chirbit:profile**
  - **Cinchcast**
  - **Cinemassacre**
- - **clipfish**
+ - **Clipfish**
  - **cliphunter**
  - **Clipsyndicate**
  - **Cloudy**
@@ -116,6 +116,7 @@
  - **DailymotionCloud**
  - **daum.net**
  - **DBTV**
+ - **DCN**
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
@@ -351,7 +352,6 @@
  - **NowTV**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
- - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
@@ -377,6 +377,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
+ - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
@@ -406,6 +407,7 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
+ - **Quickscope**: Quick Scope
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -518,6 +520,7 @@
  - **ted**
  - **TeleBruxelles**
  - **telecinco.es**
+ - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
  - **TenPlay**
@@ -621,6 +624,7 @@
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
+ - **vpro**: npo.nl and ntr.nl
  - **VRT**
  - **vube**: Vube.com
  - **VuClip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9f209499c..6462d4477 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.06.1'
+__version__ = '2015.08.09'

From c5864a8ce6379dca300f447cca12a5a946d67d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 21:38:58 +0600
Subject: [PATCH 1025/2145] [fc2] Fix python 2.6 (Closes #6512)

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 1ccc1a964..e4f7195a8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -86,7 +86,7 @@ class FC2IE(InfoExtractor):
 
         info_url = (
             "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&".
-            format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.', '%2E')))
+            format(video_id, mimi, compat_urllib_request.quote(refer, safe=b'').replace('.', '%2E')))
 
         info_webpage = self._download_webpage(
             info_url, video_id, note='Downloading info page')

From f6c3664d717857a7994f189a01a00402df2b4168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 23:35:08 +0600
Subject: [PATCH 1026/2145] [vimeo] Fix login (Closes #6488)

---
 youtube_dl/extractor/vimeo.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4c4e3c72a..5bce78ac0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -29,6 +29,7 @@ from ..utils import (
 class VimeoBaseInfoExtractor(InfoExtractor):
     _NETRC_MACHINE = 'vimeo'
     _LOGIN_REQUIRED = False
+    _LOGIN_URL = 'https://vimeo.com/log_in'
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -37,21 +38,25 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
             return
         self.report_login()
-        login_url = 'https://vimeo.com/log_in'
-        webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        webpage = self._download_webpage(self._LOGIN_URL, None, False)
+        token = self._extract_xsrft(webpage)
         data = urlencode_postdata({
+            'action': 'login',
             'email': username,
             'password': password,
-            'action': 'login',
             'service': 'vimeo',
             'token': token,
         })
-        login_request = compat_urllib_request.Request(login_url, data)
+        login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        login_request.add_header('Cookie', 'xsrft=%s' % token)
+        login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
+    def _extract_xsrft(self, webpage):
+        return self._search_regex(
+            r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
+            webpage, 'login token', group='xsrft')
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -193,7 +198,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
+        token = self._extract_xsrft(webpage)
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -422,7 +427,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         }
 
 
-class VimeoChannelIE(InfoExtractor):
+class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
@@ -453,7 +458,7 @@ class VimeoChannelIE(InfoExtractor):
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
-        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
+        token = self._extract_xsrft(webpage)
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)
@@ -603,7 +608,7 @@ class VimeoReviewIE(InfoExtractor):
         return self.url_result(player_url, 'Vimeo', video_id)
 
 
-class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
+class VimeoWatchLaterIE(VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
     _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'

From 844587669eeffaf139b9c8888550c9d3b88335e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 23:58:01 +0600
Subject: [PATCH 1027/2145] [vimeo:watchlater] Fix extraction (Closes #3886)

---
 youtube_dl/extractor/vimeo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5bce78ac0..1eeb4618e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -431,6 +431,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
+    _TITLE = None
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
     _TESTS = [{
         'url': 'https://vimeo.com/channels/tributes',
@@ -445,7 +446,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         return '%s/videos/page:%d/' % (base_url, pagenum)
 
     def _extract_list_title(self, webpage):
-        return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
+        return self._TITLE or self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
     def _login_list_password(self, page_url, list_id, webpage):
         login_form = self._search_regex(
@@ -611,11 +612,11 @@ class VimeoReviewIE(InfoExtractor):
 class VimeoWatchLaterIE(VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
-    _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'
+    _VALID_URL = r'https://vimeo\.com/(?:home/)?watchlater|:vimeowatchlater'
+    _TITLE = 'Watch Later'
     _LOGIN_REQUIRED = True
-    _TITLE_RE = r'href="/home/watchlater".*?>(.*?)<'
     _TESTS = [{
-        'url': 'https://vimeo.com/home/watchlater',
+        'url': 'https://vimeo.com/watchlater',
         'only_matching': True,
     }]
 
@@ -631,7 +632,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
         return request
 
     def _real_extract(self, url):
-        return self._extract_videos('watchlater', 'https://vimeo.com/home/watchlater')
+        return self._extract_videos('watchlater', 'https://vimeo.com/watchlater')
 
 
 class VimeoLikesIE(InfoExtractor):

From 11b5605815d685263b271b4e061c43f9cb55a08c Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puckipedia.nl>
Date: Mon, 10 Aug 2015 20:52:38 +0200
Subject: [PATCH 1028/2145] [youtube] Use the first v= argument in the URL

This is according to how youtube handles multiple v= values in one URL.
Before this, it was possible to make a single URL show up differently
 on youtube itself, and if you downloaded/viewed it with youtube-dl/mpv
---
 youtube_dl/extractor/youtube.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 67a1df9a0..eaf058cfb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -213,7 +213,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
-                                 (?:.*?&)?                                    # any other preceding param (like /?s=tuff&v=xxxx)
+                                 (?:.*?&)??                                   # any other preceding param (like /?s=tuff&v=xxxx)
                                  v=
                              )
                          ))
@@ -380,6 +380,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'setindia'
             }
         },
+        {
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
+            'note': 'Use the first video ID in the URL',
+            'info_dict': {
+                'id': 'BaW_jenozKc',
+                'ext': 'mp4',
+                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+                'uploader': 'Philipp Hagemeister',
+                'uploader_id': 'phihag',
+                'upload_date': '20121002',
+                'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
+                'categories': ['Science & Technology'],
+                'tags': ['youtube-dl'],
+                'like_count': int,
+                'dislike_count': int,
+            }
+        },
         {
             'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
             'note': '256k DASH audio (format 141) via DASH manifest',

From b29440aee64027b3e4145070b0235193752b4d9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:17:41 +0600
Subject: [PATCH 1029/2145] [vimeo:user] Do not match watchlater

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1eeb4618e..50df79ca1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -505,7 +505,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
 
 class VimeoUserIE(VimeoChannelIE):
     IE_NAME = 'vimeo:user'
-    _VALID_URL = r'https://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
+    _VALID_URL = r'https://vimeo\.com/(?!(?:[0-9]+|watchlater)(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
     _TESTS = [{
         'url': 'https://vimeo.com/nkistudio/videos',

From 34a7de2970d8bbceeb3f485d64a57f67489a44d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:22:06 +0600
Subject: [PATCH 1030/2145] [youtube] Skip download for multiple v= test

---
 youtube_dl/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eaf058cfb..01dbbfa3c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -395,7 +395,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'tags': ['youtube-dl'],
                 'like_count': int,
                 'dislike_count': int,
-            }
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',

From 34952f09e175e0b78c929fddf56f82ccf028dc5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:24:53 +0600
Subject: [PATCH 1031/2145] [youtube] Add age limit to tests

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 01dbbfa3c..e74a39095 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -365,6 +365,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'age_limit': 18,
             }
         },
         {
@@ -475,6 +476,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',
+                'age_limit': 18,
             },
         },
         # Age-gate video with encrypted signature
@@ -488,6 +490,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
                 'upload_date': '20110629',
+                'age_limit': 18,
             },
         },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)

From fb0d12c6cbcabd6f9e84d51c82dea6778d0bb863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:46:25 +0600
Subject: [PATCH 1032/2145] [pbs] Add age limit to tests

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index a53479aad..683c81de3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -92,6 +92,7 @@ class PBSIE(InfoExtractor):
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
+                'age_limit': 10,
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg

From b1ac38fadc65049dc6f9611fa7e9649de1e7eb93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:49:23 +0600
Subject: [PATCH 1033/2145] [tvplay] Add age limit to tests

---
 youtube_dl/extractor/tvplay.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 79863e781..b4683de54 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -104,6 +104,7 @@ class TVPlayIE(InfoExtractor):
                 'duration': 1492,
                 'timestamp': 1330522854,
                 'upload_date': '20120229',
+                'age_limit': 18,
             },
             'params': {
                 # rtmp download

From bf812ef71438036c23640f29bd7ae955289720ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 23:00:45 +0600
Subject: [PATCH 1034/2145] [downloader/external] Forward --proxy to wget and
 aria2c

---
 youtube_dl/downloader/external.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 07ce59f7d..49d806ee4 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,14 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _option(self, command_option, param):
+        param = self.params.get(param)
+        if param is None:
+            return []
+        if isinstance(param, bool):
+            return [command_option]
+        return [command_option, param]
+
     def _no_check_certificate(self, command_option):
         return [command_option] if self.params.get('nocheckcertificate', False) else []
 
@@ -102,6 +110,7 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._option('--proxy', 'proxy')
         cmd += self._no_check_certificate('--no-check-certificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
@@ -120,6 +129,7 @@ class Aria2cFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')
+        cmd += self._option('--all-proxy', 'proxy')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 9f3da138606773339de9accc2bc6522ea88185fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 23:05:04 +0600
Subject: [PATCH 1035/2145] [downloader/external] Use generic _option

---
 youtube_dl/downloader/external.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 49d806ee4..6c310346c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -45,12 +45,6 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
-    def _source_address(self, command_option):
-        source_address = self.params.get('source_address')
-        if source_address is None:
-            return []
-        return [command_option, source_address]
-
     def _option(self, command_option, param):
         param = self.params.get(param)
         if param is None:
@@ -59,9 +53,6 @@ class ExternalFD(FileDownloader):
             return [command_option]
         return [command_option, param]
 
-    def _no_check_certificate(self, command_option):
-        return [command_option] if self.params.get('nocheckcertificate', False) else []
-
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -88,7 +79,7 @@ class CurlFD(ExternalFD):
         cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
-        cmd += self._source_address('--interface')
+        cmd += self._option('--interface', 'source_address')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -109,9 +100,9 @@ class WgetFD(ExternalFD):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
-        cmd += self._source_address('--bind-address')
+        cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._no_check_certificate('--no-check-certificate')
+        cmd += self._option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -128,7 +119,7 @@ class Aria2cFD(ExternalFD):
         cmd += ['--out', os.path.basename(tmpfilename)]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
-        cmd += self._source_address('--interface')
+        cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
         cmd += ['--', info_dict['url']]
         return cmd

From 0a19d4ccd6914d8547fd3e42fd279c960d9f8fad Mon Sep 17 00:00:00 2001
From: sceext <sceext@foxmail.com>
Date: Wed, 12 Aug 2015 14:01:48 +0800
Subject: [PATCH 1036/2145]  [iqiyi] update md5 salt (2015-08-10 Zombie)

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index afb7f4e61..dfc6d58a0 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -201,7 +201,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        enc_key = '8e29ab5666d041c3a1ea76e06dabdffb'
+        enc_key = '3601ba290e4f4662848c710e2122007e'	# last update at 2015-08-10 for Zombie
         return enc_key
 
     def _real_extract(self, url):

From 1df3186e0e2c49993f4230ec77a9de351177b271 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 16:01:47 +0200
Subject: [PATCH 1037/2145] [funnyordie] Handle protocol-relative URLs (fixes
 #6490)

---
 youtube_dl/extractor/funnyordie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index dd87257c4..f5f13689c 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -53,7 +53,7 @@ class FunnyOrDieIE(InfoExtractor):
         for bitrate in bitrates:
             for link in links:
                 formats.append({
-                    'url': '%s%d.%s' % (link[0], bitrate, link[1]),
+                    'url': self._proto_relative_url('%s%d.%s' % (link[0], bitrate, link[1])),
                     'format_id': '%s-%d' % (link[1], bitrate),
                     'vbr': bitrate,
                 })

From f57b7835e21b00a1b2205b4bcfba50c630ff68b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 12 Aug 2015 21:27:58 +0600
Subject: [PATCH 1038/2145] [youtube] Update tests

---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e74a39095..facd837ad 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -442,7 +442,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'nfWlot6h_JM',
                 'ext': 'm4a',
                 'title': 'Taylor Swift - Shake It Off',
-                'description': 'md5:2acfda1b285bdd478ccec22f9918199d',
+                'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
@@ -515,7 +515,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
-                'upload_date': '20120731',
+                'upload_date': '20120724',
                 'uploader_id': 'olympic',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',
@@ -544,7 +544,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'qEJwOuvDf7I',
             'info_dict': {
                 'id': 'qEJwOuvDf7I',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
                 'description': '',
                 'upload_date': '20150404',

From f0f3a6c99d2834ca8af87be4978c0040c3744628 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 18:07:27 +0200
Subject: [PATCH 1039/2145] [rtvnhnl] Added new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtvnhnl.py  | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/rtvnhnl.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dad3ec87f..f026a4171 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -491,6 +491,7 @@ from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
+from .rtvnhnl import RtvnhNlIE
 from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
diff --git a/youtube_dl/extractor/rtvnhnl.py b/youtube_dl/extractor/rtvnhnl.py
new file mode 100644
index 000000000..ce84900a0
--- /dev/null
+++ b/youtube_dl/extractor/rtvnhnl.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RtvnhNlIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'params': {
+            'hls_prefer_native': True
+        },
+
+        'url': 'http://www.rtvnh.nl/video/131946',
+        'md5': '6e1d0ab079e2a00b6161442d3ceacfc1',
+        'info_dict': {
+            'id': '131946',
+            'ext': 'mp4',
+            'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
+            'thumbnail': 're:^https?://rtvnh-webfiles\.[^.]+\.amazonaws\.com/data/cache/[0-9]+/basedata/pf_image/[0-9.]+/[0-9\-a-f]+\.jpg$'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        meta = self._parse_json(self._download_webpage('http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
+        formats = self._extract_smil_formats('http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+
+        for item in meta['source']['fb']:
+            if item.get('type') == 'hls':
+                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4'))
+            elif item.get('type') == '':
+                formats.append({'url': item['file']})
+        
+        return {
+            'id': video_id,
+            'title': meta['title'].strip(),
+            'thumbnail': meta['image'],
+            'formats': formats
+        }

From fb124e37419668c34b4056575614776b0c64b401 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 20:21:32 +0200
Subject: [PATCH 1040/2145] [rtvnhnl] Relax the thumbnail check

---
 youtube_dl/extractor/rtvnhnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnhnl.py b/youtube_dl/extractor/rtvnhnl.py
index ce84900a0..0921e2648 100644
--- a/youtube_dl/extractor/rtvnhnl.py
+++ b/youtube_dl/extractor/rtvnhnl.py
@@ -17,7 +17,7 @@ class RtvnhNlIE(InfoExtractor):
             'id': '131946',
             'ext': 'mp4',
             'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
-            'thumbnail': 're:^https?://rtvnh-webfiles\.[^.]+\.amazonaws\.com/data/cache/[0-9]+/basedata/pf_image/[0-9.]+/[0-9\-a-f]+\.jpg$'
+            'thumbnail': 're:^http:.*\.jpg$'
         }
     }
 

From d9ab5262b137962995af1b444f45f7f32dc33a77 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 20:26:13 +0200
Subject: [PATCH 1041/2145] [rtvnh] Renamed rtvnhnl -> rtvnh

---
 youtube_dl/extractor/__init__.py              | 2 +-
 youtube_dl/extractor/{rtvnhnl.py => rtvnh.py} | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename youtube_dl/extractor/{rtvnhnl.py => rtvnh.py} (94%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f026a4171..9a6308723 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -491,7 +491,7 @@ from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
-from .rtvnhnl import RtvnhNlIE
+from .rtvnh import RTVNHIE
 from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
diff --git a/youtube_dl/extractor/rtvnhnl.py b/youtube_dl/extractor/rtvnh.py
similarity index 94%
rename from youtube_dl/extractor/rtvnhnl.py
rename to youtube_dl/extractor/rtvnh.py
index 0921e2648..f5c0b94a8 100644
--- a/youtube_dl/extractor/rtvnhnl.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 
-class RtvnhNlIE(InfoExtractor):
+class RTVNHIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
     _TEST = {
         'params': {
@@ -17,7 +17,7 @@ class RtvnhNlIE(InfoExtractor):
             'id': '131946',
             'ext': 'mp4',
             'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
-            'thumbnail': 're:^http:.*\.jpg$'
+            'thumbnail': 're:^https?:.*\.jpg$'
         }
     }
 

From d7dbfc7cc18c2d54d7e1752def6c4710c58b49fc Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 20:51:28 +0200
Subject: [PATCH 1042/2145] Use native HLS implementation by default.

---
 youtube_dl/extractor/rtvnh.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index f5c0b94a8..2799f01a6 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -7,10 +7,6 @@ from .common import InfoExtractor
 class RTVNHIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
     _TEST = {
-        'params': {
-            'hls_prefer_native': True
-        },
-
         'url': 'http://www.rtvnh.nl/video/131946',
         'md5': '6e1d0ab079e2a00b6161442d3ceacfc1',
         'info_dict': {
@@ -28,7 +24,7 @@ class RTVNHIE(InfoExtractor):
 
         for item in meta['source']['fb']:
             if item.get('type') == 'hls':
-                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4'))
+                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
         

From 240ca32e57a027ff8cec8617c154bb7100bead1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:00:05 +0600
Subject: [PATCH 1043/2145] [rtvnh] Carry long lines

---
 youtube_dl/extractor/rtvnh.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 2799f01a6..998a3c53d 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -19,12 +19,16 @@ class RTVNHIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        meta = self._parse_json(self._download_webpage('http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
-        formats = self._extract_smil_formats('http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+
+        meta = self._parse_json(self._download_webpage(
+            'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
+        formats = self._extract_smil_formats(
+            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
 
         for item in meta['source']['fb']:
             if item.get('type') == 'hls':
-                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
+                formats.extend(self._extract_m3u8_formats(
+                    item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
         

From f196047832a2da74d5adf75759877b5d95ec5b5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:00:25 +0600
Subject: [PATCH 1044/2145] [rtvnh] Make thumbnail optional

---
 youtube_dl/extractor/rtvnh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 998a3c53d..d576a3410 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -35,6 +35,6 @@ class RTVNHIE(InfoExtractor):
         return {
             'id': video_id,
             'title': meta['title'].strip(),
-            'thumbnail': meta['image'],
+            'thumbnail': meta.get('image'),
             'formats': formats
         }

From 60231c65b9a50e08967d748c3ed401488fed3587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:02:50 +0600
Subject: [PATCH 1045/2145] [rtvnh] Make SMIL not fatal

---
 youtube_dl/extractor/rtvnh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index d576a3410..202ea0181 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -23,7 +23,7 @@ class RTVNHIE(InfoExtractor):
         meta = self._parse_json(self._download_webpage(
             'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
         formats = self._extract_smil_formats(
-            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id, fatal=False)
 
         for item in meta['source']['fb']:
             if item.get('type') == 'hls':

From 2c919adb74893544ab6def1d56ff8ed37c282ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:11:55 +0600
Subject: [PATCH 1046/2145] [rtvnh] Check status code

---
 youtube_dl/extractor/rtvnh.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 202ea0181..7c9d4b0cd 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class RTVNHIE(InfoExtractor):
@@ -22,6 +23,12 @@ class RTVNHIE(InfoExtractor):
 
         meta = self._parse_json(self._download_webpage(
             'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
+
+        status = meta.get('status')
+        if status != 200:
+            raise ExtractorError(
+                '%s returned error code %d' % (self.IE_NAME, status), expected=True)
+
         formats = self._extract_smil_formats(
             'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id, fatal=False)
 
@@ -31,7 +38,7 @@ class RTVNHIE(InfoExtractor):
                     item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
-        
+
         return {
             'id': video_id,
             'title': meta['title'].strip(),

From 3b7130439aade87b628fa6dd727df5860323a68f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:15:58 +0600
Subject: [PATCH 1047/2145] Credit @ngld for RTVNH (#6537)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d16d34272..71c420165 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -137,3 +137,4 @@ Zach Bruggeman
 Tjark Saul
 slangangular
 Behrouz Abbasi
+ngld

From b6b2711298f8d43414deac939f92c7c3477826b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:17:15 +0600
Subject: [PATCH 1048/2145] [tweakers] Remove unused imports

---
 youtube_dl/extractor/tweakers.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 6eeffb1cc..f3198fb85 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,12 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    xpath_with_ns,
-    int_or_none,
-    float_or_none,
-)
 
 
 class TweakersIE(InfoExtractor):

From e73c85cb23d278702357412479fd4b162a3abbb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:18:49 +0600
Subject: [PATCH 1049/2145] [iqiyi] PEP 8

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index dfc6d58a0..393e67e35 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -201,7 +201,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        enc_key = '3601ba290e4f4662848c710e2122007e'	# last update at 2015-08-10 for Zombie
+        enc_key = '3601ba290e4f4662848c710e2122007e'  # last update at 2015-08-10 for Zombie
         return enc_key
 
     def _real_extract(self, url):

From 237c03c8eaa4da1713a635e87f98ac14430b35cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:19:23 +0600
Subject: [PATCH 1050/2145] [dhm] Remove unused import

---
 youtube_dl/extractor/dhm.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 127eb0439..44e0c5d4d 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -1,10 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    parse_duration,
-)
+from ..utils import parse_duration
 
 
 class DHMIE(InfoExtractor):

From 28479149ccf3425e6a6e35d3a155f6802629728a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 13 Aug 2015 12:56:12 +0800
Subject: [PATCH 1051/2145] [theplatform] Fallback to hardcoded releaseUrl if
 not available

Fixes #6546.

Not adding a test case as test_NBC has the same problem.
---
 youtube_dl/extractor/theplatform.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 83d833e30..0643eccaf 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -108,7 +108,11 @@ class ThePlatformIE(InfoExtractor):
             config_url = config_url.replace('swf/', 'config/')
             config_url = config_url.replace('onsite/', 'onsite/config/')
             config = self._download_json(config_url, video_id, 'Downloading config')
-            smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
+            if 'releaseUrl' in config:
+                release_url = config['releaseUrl']
+            else:
+                release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
+            smil_url = release_url + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
             smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
 

From 6828c809e44fca7b19da3c62a11cea313a86b64e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 21:07:14 +0600
Subject: [PATCH 1052/2145] [downloader/fragment] Respect --retries for
 fragment based downloaders (Closes #6549)

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 5f9d6796d..5a64b29ee 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -35,6 +35,7 @@ class FragmentFD(FileDownloader):
                 'quiet': True,
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit', None),
+                'retries': self.params.get('retries', 0),
                 'test': self.params.get('test', False),
             }
         )

From 7393746da213bec686f8425165854e5e383b7eb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 21:10:11 +0600
Subject: [PATCH 1053/2145] [downloader/hls] Add _debug_cmd

---
 youtube_dl/downloader/hls.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 60dca0ab1..2b6c3370f 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -32,6 +32,8 @@ class HlsFD(FileDownloader):
             for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
+        self._debug_cmd(args)
+
         retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))

From cb28e0338665c96b2d5b35d203b1d54a57f3feb1 Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@openmailbox.org>
Date: Mon, 10 Aug 2015 19:27:16 +0200
Subject: [PATCH 1054/2145] [indavideo] Add new extractor

Closes #2147.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/indavideo.py | 79 +++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/indavideo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9a6308723..3bcfa93bb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -242,6 +242,7 @@ from .imdb import (
 )
 from .imgur import ImgurIE
 from .ina import InaIE
+from .indavideo import IndavideoIE
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
new file mode 100644
index 000000000..2a2cf2bd3
--- /dev/null
+++ b/youtube_dl/extractor/indavideo.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .. import utils
+from .common import InfoExtractor
+
+
+class IndavideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>.+)'
+    _TESTS = [
+        {
+            'url': 'http://indavideo.hu/video/Cicatanc',
+            'md5': 'c8a507a1c7410685f83a06eaeeaafeab',
+            'info_dict': {
+                'id': '1837039',
+                'title': 'Cicatánc',
+                'ext': 'mp4',
+                'display_id': 'Cicatanc',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': '',
+                'uploader': 'cukiajanlo',
+                'uploader_id': '83729',
+                'duration': 72,
+                'age_limit': 0,
+                'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom']
+            },
+        },
+        {
+            'url': 'http://indavideo.hu/video/Vicces_cica_1',
+            'md5': '8c82244ba85d2a2310275b318eb51eac',
+            'info_dict': {
+                'id': '1335611',
+                'title': 'Vicces cica',
+                'ext': 'mp4',
+                'display_id': 'Vicces_cica_1',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Játszik a tablettel. :D',
+                'uploader': 'Jet_Pack',
+                'uploader_id': '491217',
+                'duration': 7,
+                'age_limit': 0,
+                'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_disp_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_disp_id)
+
+        embed_url = self._html_search_regex(r'<link rel="video_src" href="(.+?)"/>', webpage, 'embed_url')
+        video_hash = embed_url.split('/')[-1]
+
+        payload = self._download_json('http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/' + video_hash, video_disp_id)
+        video_info = payload['data']
+
+        thumbnails = video_info.get('thumbnails')
+        if thumbnails:
+            thumbnails = [{'url': self._proto_relative_url(x)} for x in thumbnails]
+
+        tags = video_info.get('tags')
+        if tags:
+            tags = [x['title'] for x in tags]
+
+        return {
+            'id': video_info.get('id'),
+            'title': video_info['title'],
+            'url': video_info['video_file'],
+            'ext': 'mp4',
+            'display_id': video_disp_id,
+            'thumbnails': thumbnails,
+            'description': video_info.get('description'),
+            'uploader': video_info.get('user_name'),
+            # TODO: upload date (it's in CET/CEST)
+            'uploader_id': video_info.get('user_id'),
+            'duration': utils.int_or_none(video_info.get('length')),
+            'age_limit': utils.int_or_none(video_info.get('age_limit')),
+            'tags': tags,
+        }

From 3c12a027d48a2d6d1162ab515df0308237aef881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:25:47 +0600
Subject: [PATCH 1055/2145] [indavideo] Split in two extractors, extract all
 formats and fix timestamp

---
 youtube_dl/extractor/__init__.py  |   5 +-
 youtube_dl/extractor/indavideo.py | 178 +++++++++++++++++++-----------
 2 files changed, 118 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3bcfa93bb..83d21bd15 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -242,7 +242,10 @@ from .imdb import (
 )
 from .imgur import ImgurIE
 from .ina import InaIE
-from .indavideo import IndavideoIE
+from .indavideo import (
+    IndavideoIE,
+    IndavideoEmbedIE,
+)
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 2a2cf2bd3..b75715244 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -3,77 +3,127 @@ from __future__ import unicode_literals
 
 from .. import utils
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
+)
+
+
+class IndavideoEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:embed\.)?indavideo\.hu/player/video/|assets\.indavideo\.hu/swf/player\.swf\?.*\b(?:v(?:ID|id))=)(?P<id>[\da-f]+)'
+    _TESTS = [{
+        'url': 'http://indavideo.hu/player/video/1bdc3c6d80/',
+        'md5': 'f79b009c66194acacd40712a6778acfa',
+        'info_dict': {
+            'id': '1837039',
+            'ext': 'mp4',
+            'title': 'Cicatánc',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'cukiajanlo',
+            'uploader_id': '83729',
+            'timestamp': 1439193826,
+            'upload_date': '20150810',
+            'duration': 72,
+            'age_limit': 0,
+            'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom'],
+        },
+    }, {
+        'url': 'http://embed.indavideo.hu/player/video/1bdc3c6d80?autostart=1&hide=1',
+        'only_matching': True,
+    }, {
+        'url': 'http://assets.indavideo.hu/swf/player.swf?v=fe25e500&vID=1bdc3c6d80&autostart=1&hide=1&i=1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/%s' % video_id,
+            video_id)['data']
+
+        video_id = video['id']
+        title = video['title']
+
+        video_urls = video.get('video_files', [])
+        video_file = video.get('video_file')
+        if video:
+            video_urls.append(video_file)
+        video_urls = list(set(video_urls))
+
+        video_prefix = video_urls[0].rsplit('/', 1)[0]
+
+        for flv_file in video.get('flv_files', []):
+            flv_url = '%s/%s' % (video_prefix, flv_file)
+            if flv_url not in video_urls:
+                video_urls.append(flv_url)
+
+        formats = [{
+            'url': video_url,
+            'height': self._search_regex(r'\.(\d{3,4})\.mp4$', video_url, 'height', default=None),
+        } for video_url in video_urls]
+        self._sort_formats(formats)
+
+        timestamp = video.get('date')
+        if timestamp:
+            # upload date is in CEST
+            timestamp = parse_iso8601(timestamp + ' +0200', ' ')
+
+        thumbnails = [{
+            'url': self._proto_relative_url(thumbnail)
+        } for thumbnail in video.get('thumbnails', [])]
+
+        tags = [tag['title'] for tag in video.get('tags', [])]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video.get('description'),
+            'thumbnails': thumbnails,
+            'uploader': video.get('user_name'),
+            'uploader_id': video.get('user_id'),
+            'timestamp': timestamp,
+            'duration': int_or_none(video.get('length')),
+            'age_limit': parse_age_limit(video.get('age_limit')),
+            'tags': tags,
+            'formats': formats,
+        }
 
 
 class IndavideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>.+)'
-    _TESTS = [
-        {
-            'url': 'http://indavideo.hu/video/Cicatanc',
-            'md5': 'c8a507a1c7410685f83a06eaeeaafeab',
-            'info_dict': {
-                'id': '1837039',
-                'title': 'Cicatánc',
-                'ext': 'mp4',
-                'display_id': 'Cicatanc',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': '',
-                'uploader': 'cukiajanlo',
-                'uploader_id': '83729',
-                'duration': 72,
-                'age_limit': 0,
-                'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom']
-            },
+    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
+    _TEST = {
+        'url': 'http://indavideo.hu/video/Vicces_cica_1',
+        'md5': '8c82244ba85d2a2310275b318eb51eac',
+        'info_dict': {
+            'id': '1335611',
+            'display_id': 'Vicces_cica_1',
+            'ext': 'mp4',
+            'title': 'Vicces cica',
+            'description': 'Játszik a tablettel. :D',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Jet_Pack',
+            'uploader_id': '491217',
+            'timestamp': 1390821212,
+            'upload_date': '20140127',
+            'duration': 7,
+            'age_limit': 0,
+            'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
         },
-        {
-            'url': 'http://indavideo.hu/video/Vicces_cica_1',
-            'md5': '8c82244ba85d2a2310275b318eb51eac',
-            'info_dict': {
-                'id': '1335611',
-                'title': 'Vicces cica',
-                'ext': 'mp4',
-                'display_id': 'Vicces_cica_1',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'Játszik a tablettel. :D',
-                'uploader': 'Jet_Pack',
-                'uploader_id': '491217',
-                'duration': 7,
-                'age_limit': 0,
-                'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
-            },
-        },
-    ]
+    }
 
     def _real_extract(self, url):
-        video_disp_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_disp_id)
+        display_id = self._match_id(url)
 
-        embed_url = self._html_search_regex(r'<link rel="video_src" href="(.+?)"/>', webpage, 'embed_url')
-        video_hash = embed_url.split('/')[-1]
-
-        payload = self._download_json('http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/' + video_hash, video_disp_id)
-        video_info = payload['data']
-
-        thumbnails = video_info.get('thumbnails')
-        if thumbnails:
-            thumbnails = [{'url': self._proto_relative_url(x)} for x in thumbnails]
-
-        tags = video_info.get('tags')
-        if tags:
-            tags = [x['title'] for x in tags]
+        webpage = self._download_webpage(url, display_id)
+        embed_url = self._search_regex(
+            r'<link[^>]+rel="video_src"[^>]+href="(.+?)"', webpage, 'embed url')
 
         return {
-            'id': video_info.get('id'),
-            'title': video_info['title'],
-            'url': video_info['video_file'],
-            'ext': 'mp4',
-            'display_id': video_disp_id,
-            'thumbnails': thumbnails,
-            'description': video_info.get('description'),
-            'uploader': video_info.get('user_name'),
-            # TODO: upload date (it's in CET/CEST)
-            'uploader_id': video_info.get('user_id'),
-            'duration': utils.int_or_none(video_info.get('length')),
-            'age_limit': utils.int_or_none(video_info.get('age_limit')),
-            'tags': tags,
+            '_type': 'url_transparent',
+            'ie_key': 'IndavideoEmbed',
+            'url': embed_url,
+            'display_id': display_id,
         }

From a34e19629c407a08cd9065223f26f1f5468a4423 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:40:20 +0600
Subject: [PATCH 1056/2145] [indavideo] Relax _VALID_URL to match subdomains
 and add tests

---
 youtube_dl/extractor/indavideo.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index b75715244..550a7001b 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -44,7 +44,6 @@ class IndavideoEmbedIE(InfoExtractor):
             'http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/%s' % video_id,
             video_id)['data']
 
-        video_id = video['id']
         title = video['title']
 
         video_urls = video.get('video_files', [])
@@ -78,7 +77,7 @@ class IndavideoEmbedIE(InfoExtractor):
         tags = [tag['title'] for tag in video.get('tags', [])]
 
         return {
-            'id': video_id,
+            'id': video.get('id') or video_id,
             'title': title,
             'description': video.get('description'),
             'thumbnails': thumbnails,
@@ -93,8 +92,8 @@ class IndavideoEmbedIE(InfoExtractor):
 
 
 class IndavideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:.+?\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
+    _TESTS = [{
         'url': 'http://indavideo.hu/video/Vicces_cica_1',
         'md5': '8c82244ba85d2a2310275b318eb51eac',
         'info_dict': {
@@ -112,7 +111,22 @@ class IndavideoIE(InfoExtractor):
             'age_limit': 0,
             'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
         },
-    }
+    }, {
+        'url': 'http://index.indavideo.hu/video/2015_0728_beregszasz',
+        'only_matching': True,
+    }, {
+        'url': 'http://auto.indavideo.hu/video/Sajat_utanfutoban_a_kis_tacsko',
+        'only_matching': True,
+    }, {
+        'url': 'http://erotika.indavideo.hu/video/Amator_tini_punci',
+        'only_matching': True,
+    }, {
+        'url': 'http://film.indavideo.hu/video/f_hrom_nagymamm_volt',
+        'only_matching': True,
+    }, {
+        'url': 'http://palyazat.indavideo.hu/video/Embertelen_dal_Dodgem_egyuttes',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From fb56131dd9cf3bfa31d7d6920a135281d151f803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:47:12 +0600
Subject: [PATCH 1057/2145] Credit @nyuszika7h for indavideo (#6517)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 71c420165..ded9e87d2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -138,3 +138,4 @@ Tjark Saul
 slangangular
 Behrouz Abbasi
 ngld
+nyuszika7h

From 594f51b85934878ff20b608f312d7f564e3a3d71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:47:49 +0600
Subject: [PATCH 1058/2145] [indavideo] Remove unused import

---
 youtube_dl/extractor/indavideo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 550a7001b..12fb5e8e1 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .. import utils
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,

From 3cafca04aaf2bfc4d31e8255b9cb75e8f1ad4b16 Mon Sep 17 00:00:00 2001
From: reddraggone9 <cljenkins9@gmail.com>
Date: Fri, 14 Aug 2015 00:35:35 -0500
Subject: [PATCH 1059/2145] Updated line numbers in the fragment portion of
 README links.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 15baf75ce..e91119d84 100644
--- a/README.md
+++ b/README.md
@@ -544,7 +544,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
@@ -572,7 +572,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L92). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From 4d2ad866f347086d3a1cf4cb7e0a8cadd3c87748 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 14 Aug 2015 19:18:03 +0800
Subject: [PATCH 1060/2145] [README.md] Document format_id field in output
 template section (#6557)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 15baf75ce..8fa402ee2 100644
--- a/README.md
+++ b/README.md
@@ -272,6 +272,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
  - `playlist`: The name or the id of the playlist that contains the video.
  - `playlist_index`: The index of the video in the playlist, a five-digit number.
+ - `format_id`: The sequence will be replaced by the format code specified by `--format`.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 

From 41dbc50f9c7dfaad4084fbeac77192c7ac37daca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Aug 2015 22:07:02 +0600
Subject: [PATCH 1061/2145] [lynda] Capture and output login error (Closes
 #6556)

---
 youtube_dl/extractor/lynda.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index deead220a..5b9157ed4 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    clean_html,
     int_or_none,
 )
 
@@ -70,6 +71,15 @@ class LyndaBaseIE(InfoExtractor):
                     'Confirming log in and log out from another device')
 
         if all(not re.search(p, login_page) for p in ('isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
+            if 'login error' in login_page:
+                mobj = re.search(
+                    r'(?s)<h1[^>]+class="topmost">(?P<title>[^<]+)</h1>\s*<div>(?P<description>.+?)</div>',
+                    login_page)
+                if mobj:
+                    raise ExtractorError(
+                        'lynda returned error: %s - %s'
+                        % (mobj.group('title'), clean_html(mobj.group('description'))),
+                        expected=True)
             raise ExtractorError('Unable to log in')
 
 

From 6be5e46994ea5db76d7a2659260606898c265957 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 14 Aug 2015 22:22:39 +0600
Subject: [PATCH 1062/2145] [README.md] Clarify line ranges

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e52cdb941..542a7c26a 100644
--- a/README.md
+++ b/README.md
@@ -545,7 +545,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
@@ -573,7 +573,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L92). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From d0d6c097fc7859180f16a445536029c600b1e57f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 15 Aug 2015 15:17:27 +0800
Subject: [PATCH 1063/2145] [moniker] Support embed- URLs (#6450)

---
 youtube_dl/extractor/moniker.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 88dcd4f73..69e4bcd1a 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -9,7 +9,10 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    remove_start,
+)
 
 
 class MonikerIE(InfoExtractor):
@@ -24,6 +27,14 @@ class MonikerIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'youtube-dl test video',
         },
+    }, {
+        'url': 'http://allmyvideos.net/embed-jih3nce3x6wn',
+        'md5': '710883dee1bfc370ecf9fa6a89307c88',
+        'info_dict': {
+            'id': 'jih3nce3x6wn',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video',
+        },
     }, {
         'url': 'http://vidspot.net/l2ngsmhs8ci5',
         'md5': '710883dee1bfc370ecf9fa6a89307c88',
@@ -38,7 +49,10 @@ class MonikerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        orig_video_id = self._match_id(url)
+        video_id = remove_start(orig_video_id, 'embed-')
+        url = url.replace(orig_video_id, video_id)
+        assert re.match(self._VALID_URL, url) is not None
         orig_webpage = self._download_webpage(url, video_id)
 
         if '>File Not Found<' in orig_webpage:

From 8b8c1093b65ee02aad859ed8d82217312ed0d9d8 Mon Sep 17 00:00:00 2001
From: Shaun Walbridge <shaun.walbridge@gmail.com>
Date: Sat, 18 Apr 2015 00:37:04 -0400
Subject: [PATCH 1064/2145] [EsriVideo] Add new extractor

Add extractor for [videos.esri.com](https://videos.esri.com), a collection
of videos relating to GIS.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/videoesri.py | 90 +++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 youtube_dl/extractor/videoesri.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 83d21bd15..a4387636f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -695,6 +695,7 @@ from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
+from .videoesri import VideoEsriIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
diff --git a/youtube_dl/extractor/videoesri.py b/youtube_dl/extractor/videoesri.py
new file mode 100644
index 000000000..0f84323a4
--- /dev/null
+++ b/youtube_dl/extractor/videoesri.py
@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    unified_strdate
+)
+
+
+class VideoEsriIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://video.esri.com/watch/4228',
+        'md5': '170b4d513c2466ed483c150a48384133',
+        'info_dict': {
+            'id': '4228',
+            'ext': 'mp4',
+            'title': 'AppStudio for ArcGIS',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150310',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+
+        upload_date_raw = self._search_regex(
+            r'http-equiv="last-modified" content="(.*)"',
+            webpage, 'upload date')
+        upload_date = unified_strdate(upload_date_raw)
+
+        settings_info = self._search_regex(
+            r'evPlayerSettings = {(.*?);\s*$',
+            webpage, 'settings info', flags=re.MULTILINE | re.DOTALL)
+
+        # thumbnail includes '_x' for large, also has {_m,_t,_s} or
+        # without size suffix returns full image
+        thumbnail_path = re.findall(
+            r'image\': \'(\/thumbs.*)\'',
+            settings_info)[0]
+
+        if thumbnail_path:
+            thumbnail = '/'.join(['http://video.esri.com', thumbnail_path])
+
+        # note that this misses the (exceedly rare) webm files
+        video_paths = re.findall(r'mp4:(.*)\'', settings_info)
+
+        # find possible http servers of the mp4 files (also has rtsp)
+        base_url = re.findall(
+            r'netstreambasepath\':\s\'(h.*)\'', settings_info)[0]
+
+        # these are the numbers used internally, but really map
+        # to other resolutions, e.g. 960 is 720p.
+        heights = [480, 720, 960]
+        videos_by_res = {}
+        for video_path in video_paths:
+            url = "{base_url}{video_path}".format(
+                base_url=base_url,
+                video_path=video_path)
+            filename, ext = os.path.splitext(video_path)
+            height_label = int(filename.split('_')[1])
+            videos_by_res[height_label] = {
+                'url': url,
+                'ext': ext[1:],
+                'protocol': 'http',  # http-only supported currently
+            }
+
+        formats = []
+        for height in heights:
+            if height in videos_by_res:
+                formats.append(videos_by_res[height])
+
+        result = {
+            'id': video_id,
+            'title': title,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
+
+        if thumbnail:
+            result['thumbnail'] = thumbnail
+
+        return result

From 8b9848ac5678356757f67a412f7ed89a0f559be7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 15:58:30 +0600
Subject: [PATCH 1065/2145] [extractor/common] Expand meta regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5982055be..16ae4b98f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -640,7 +640,7 @@ class InfoExtractor(object):
     @staticmethod
     def _meta_regex(prop):
         return r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property|id)=(["\']?)%s\1)
+                    (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 
     def _og_search_property(self, prop, html, name=None, **kargs):

From 3aa697f993e3719cf032c5b1e192a034100b0534 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 15:58:56 +0600
Subject: [PATCH 1066/2145] [esri:video] Extract all formats and simplify

---
 youtube_dl/extractor/videoesri.py | 106 +++++++++++++-----------------
 1 file changed, 45 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/videoesri.py b/youtube_dl/extractor/videoesri.py
index 0f84323a4..84faba678 100644
--- a/youtube_dl/extractor/videoesri.py
+++ b/youtube_dl/extractor/videoesri.py
@@ -1,90 +1,74 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import os
 import re
 
 from .common import InfoExtractor
-
+from ..compat import compat_urlparse
 from ..utils import (
-    unified_strdate
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
 )
 
 
 class VideoEsriIE(InfoExtractor):
     _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'https://video.esri.com/watch/4228',
-        'md5': '170b4d513c2466ed483c150a48384133',
+        'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',
+        'md5': 'd4aaf1408b221f1b38227a9bbaeb95bc',
         'info_dict': {
-            'id': '4228',
+            'id': '1124',
             'ext': 'mp4',
-            'title': 'AppStudio for ArcGIS',
+            'title': 'ArcGIS Online - Developing Applications',
+            'description': 'Jeremy Bartley demonstrates how to develop applications with ArcGIS Online.',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'upload_date': '20150310',
+            'duration': 185,
+            'upload_date': '20120419',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
-
-        upload_date_raw = self._search_regex(
-            r'http-equiv="last-modified" content="(.*)"',
-            webpage, 'upload date')
-        upload_date = unified_strdate(upload_date_raw)
-
-        settings_info = self._search_regex(
-            r'evPlayerSettings = {(.*?);\s*$',
-            webpage, 'settings info', flags=re.MULTILINE | re.DOTALL)
-
-        # thumbnail includes '_x' for large, also has {_m,_t,_s} or
-        # without size suffix returns full image
-        thumbnail_path = re.findall(
-            r'image\': \'(\/thumbs.*)\'',
-            settings_info)[0]
-
-        if thumbnail_path:
-            thumbnail = '/'.join(['http://video.esri.com', thumbnail_path])
-
-        # note that this misses the (exceedly rare) webm files
-        video_paths = re.findall(r'mp4:(.*)\'', settings_info)
-
-        # find possible http servers of the mp4 files (also has rtsp)
-        base_url = re.findall(
-            r'netstreambasepath\':\s\'(h.*)\'', settings_info)[0]
-
-        # these are the numbers used internally, but really map
-        # to other resolutions, e.g. 960 is 720p.
-        heights = [480, 720, 960]
-        videos_by_res = {}
-        for video_path in video_paths:
-            url = "{base_url}{video_path}".format(
-                base_url=base_url,
-                video_path=video_path)
-            filename, ext = os.path.splitext(video_path)
-            height_label = int(filename.split('_')[1])
-            videos_by_res[height_label] = {
-                'url': url,
-                'ext': ext[1:],
-                'protocol': 'http',  # http-only supported currently
-            }
-
         formats = []
-        for height in heights:
-            if height in videos_by_res:
-                formats.append(videos_by_res[height])
+        for width, height, content in re.findall(
+                r'(?s)<li><strong>(\d+)x(\d+):</strong>(.+?)</li>', webpage):
+            for video_url, ext, filesize in re.findall(
+                    r'<a[^>]+href="([^"]+)">([^<]+)&nbsp;\(([^<]+)\)</a>', content):
+                formats.append({
+                    'url': compat_urlparse.urljoin(url, video_url),
+                    'ext': ext.lower(),
+                    'format_id': '%s-%s' % (ext.lower(), height),
+                    'width': int(width),
+                    'height': int(height),
+                    'filesize_approx': parse_filesize(filesize),
+                })
+        self._sort_formats(formats)
 
-        result = {
+        title = self._html_search_meta('title', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
+        if thumbnail:
+            thumbnail = re.sub(r'_[st]\.jpg$', '_x.jpg', thumbnail)
+
+        duration = int_or_none(self._search_regex(
+            [r'var\s+videoSeconds\s*=\s*(\d+)', r"'duration'\s*:\s*(\d+)"],
+            webpage, 'duration', fatal=False))
+
+        upload_date = unified_strdate(self._html_search_meta(
+            'last-modified', webpage, 'upload date', fatal=None))
+
+        return {
             'id': video_id,
             'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'upload_date': upload_date,
-            'formats': formats,
+            'formats': formats
         }
-
-        if thumbnail:
-            result['thumbnail'] = thumbnail
-
-        return result

From 9c21f229236c77a8865c857b43c6cbd95dcc6f23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 15:59:35 +0600
Subject: [PATCH 1067/2145] [esri:video] Rename extractor

---
 youtube_dl/extractor/__init__.py               | 2 +-
 youtube_dl/extractor/{videoesri.py => esri.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{videoesri.py => esri.py} (98%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4387636f..760f73918 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -695,7 +695,7 @@ from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
-from .videoesri import VideoEsriIE
+from .esri import EsriVideoIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
diff --git a/youtube_dl/extractor/videoesri.py b/youtube_dl/extractor/esri.py
similarity index 98%
rename from youtube_dl/extractor/videoesri.py
rename to youtube_dl/extractor/esri.py
index 84faba678..bf5d2019f 100644
--- a/youtube_dl/extractor/videoesri.py
+++ b/youtube_dl/extractor/esri.py
@@ -12,7 +12,7 @@ from ..utils import (
 )
 
 
-class VideoEsriIE(InfoExtractor):
+class EsriVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',

From 1d25e9d173931da0d2cb65b114f44bbf24184f6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 16:00:24 +0600
Subject: [PATCH 1068/2145] [extractor/__init__] Fix order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 760f73918..a8be63624 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -158,6 +158,7 @@ from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .espn import ESPNIE
+from .esri import EsriVideoIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
@@ -695,7 +696,6 @@ from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
-from .esri import EsriVideoIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE

From fab83e24567226fa70e7f5076d961b83239ccfbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 16:10:20 +0600
Subject: [PATCH 1069/2145] Credit @scw for video.esri.com (#5459)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ded9e87d2..d1693224e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -139,3 +139,4 @@ slangangular
 Behrouz Abbasi
 ngld
 nyuszika7h
+Shaun Walbridge

From c576ef1e7cfd31ca94ca6025c054b3ae4f611b21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 18:13:37 +0600
Subject: [PATCH 1070/2145] [shahid] Improve and simplify

---
 youtube_dl/extractor/shahid.py | 140 ++++++++++++++++++---------------
 1 file changed, 78 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 399140189..6e9903d5e 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -2,90 +2,106 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
-    js_to_json,
     ExtractorError,
-    int_or_none
+    int_or_none,
+    parse_iso8601,
 )
 
 
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
-    _TESTS = [
-        {
-            'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
-            'info_dict': {
-                'id': '90574',
-                'ext': 'm3u8',
-                'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
-                'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
-                'duration': 2972,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            }
+    _TESTS = [{
+        'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
+        'info_dict': {
+            'id': '90574',
+            'ext': 'm3u8',
+            'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
+            'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
+            'duration': 2972,
+            'timestamp': 1422057420,
+            'upload_date': '20150123',
         },
-        {
-            # shahid plus subscriber only
-            'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
-            'only_matching': True
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         }
-    ]
+    }, {
+        # shahid plus subscriber only
+        'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+        'only_matching': True
+    }]
 
-    _api_vars = {
-        'type': 'player',
-        'url': 'http://api.shahid.net/api/v1_1',
-        'playerType': 'episode',
-    }
+    def _handle_error(self, response):
+        if not isinstance(response, dict):
+            return
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
+                expected=True)
+
+    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
+        response = super(ShahidIE, self)._download_json(url, video_id, note)['data']
+        self._handle_error(response)
+        return response
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        player_info = ''
-        flash_vars = self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars', None)
-        if flash_vars is not None:
-            for line in flash_vars.splitlines():
-                if '+' not in line and '(' not in line and ')' not in line:
-                    player_info += line
-            player_info = self._parse_json(player_info, video_id, js_to_json, False)
-            if player_info is not None:
-                for key in self._api_vars:
-                    if key in player_info:
-                        self._api_vars[key] = player_info[key]
+        api_vars = {
+            'id': video_id,
+            'type': 'player',
+            'url': 'http://api.shahid.net/api/v1_1',
+            'playerType': 'episode',
+        }
 
-        player_json_data = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + self._api_vars['type'] + '.html',
-            video_id
-        )['data']
-        if 'url' in player_json_data:
-            m3u8_url = player_json_data['url']
-        else:
-            for error in player_json_data['error'].values():
-                raise ExtractorError(error)
-        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        flashvars = self._search_regex(
+            r'var\s+flashvars\s*=\s*({[^}]+})', webpage, 'flashvars', default=None)
+        if flashvars:
+            for key in api_vars.keys():
+                value = self._search_regex(
+                    r'\b%s\s*:\s*(?P<q>["\'])(?P<value>.+?)(?P=q)' % key,
+                    flashvars, 'type', default=None, group='value')
+                if value:
+                    api_vars[key] = value
 
-        video_info = self._download_json(
-            self._api_vars['url'] + '/' + self._api_vars['playerType'] + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
-            video_id
-        )['data']
-        if video_info.get('error'):
-            for error in video_info['error']:
-                raise ExtractorError(error)
-        video_info = video_info[self._api_vars['playerType']]
-        title = video_info['title']
-        thumbnail = video_info.get('thumbnailUrl')
-        categories = [category['name'] for category in video_info.get('genres')]
-        description = video_info.get('description')
-        duration = int_or_none(video_info.get('duration'))
+        player = self._download_json(
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-%s.html'
+            % (video_id, api_vars['type']), video_id, 'Downloading player JSON')
+
+        formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
+
+        video = self._download_json(
+            '%s/%s/%s?%s' % (
+                api_vars['url'], api_vars['playerType'], api_vars['id'],
+                compat_urllib_parse.urlencode({
+                    'apiKey': 'sh@hid0nlin3',
+                    'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
+                }).encode('utf-8')),
+            video_id, 'Downloading video JSON')
+
+        video = video[api_vars['playerType']]
+
+        title = video['title']
+        description = video.get('description')
+        thumbnail = video.get('thumbnailUrl')
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('referenceDate'))
+        categories = [
+            category['name']
+            for category in video.get('genres', []) if 'name' in category]
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
-            'categories': categories,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
+            'timestamp': timestamp,
+            'categories': categories,
             'formats': formats,
         }

From 9303ce3e6969b5818982d6214a8d0ff4e3c95286 Mon Sep 17 00:00:00 2001
From: reddraggone9 <cljenkins9@gmail.com>
Date: Thu, 13 Aug 2015 22:11:11 -0500
Subject: [PATCH 1071/2145] [youtube] Fix two-factor authentication

---
 youtube_dl/extractor/youtube.py | 40 +++++++++++++++------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index facd837ad..bfa9a12a8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -46,7 +46,7 @@ from ..utils import (
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
-    _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor'
+    _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
@@ -128,7 +128,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         # Two-Factor
         # TODO add SMS and phone call support - these require making a request and then prompting the user
 
-        if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', login_results) is not None:
+        if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
             tfa_code = self._get_tfa_info()
 
             if tfa_code is None:
@@ -136,31 +136,27 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 self._downloader.report_warning('(Note that only TOTP (Google Authenticator App) codes work at this time.)')
                 return False
 
-            # Unlike the first login form, secTok and timeStmp are both required for the TFA form
+            def find_value(element_id):
+                match = re.search(r'id="%s"\s+value="(.+?)">' % element_id, login_results, re.M | re.U)
+                if match is None:
+                    self._downloader.report_warning('Failed to get %s - did the page structure change?' % id)
+                return match.group(1)
 
-            match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
-            if match is None:
-                self._downloader.report_warning('Failed to get secTok - did the page structure change?')
-            secTok = match.group(1)
-            match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
-            if match is None:
-                self._downloader.report_warning('Failed to get timeStmp - did the page structure change?')
-            timeStmp = match.group(1)
+            challengeId = find_value('challengeId')
+            challengeType = find_value('challengeType')
+            gxf = find_value('gxf')
 
             tfa_form_strs = {
+                'challengeId': challengeId,
+                'challengeType': challengeType,  # This doesn't appear to change
                 'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
-                'smsToken': '',
-                'smsUserPin': tfa_code,
-                'smsVerifyPin': 'Verify',
-
-                'PersistentCookie': 'yes',
-                'checkConnection': '',
-                'checkedDomains': 'youtube',
-                'pstMsg': '1',
-                'secTok': secTok,
-                'timeStmp': timeStmp,
                 'service': 'youtube',
                 'hl': 'en_US',
+                'checkedDomains': 'youtube',
+                'pstMsg': '0',
+                'gxf': gxf,
+                'Pin': tfa_code,
+                'TrustDevice': 'on',
             }
             tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
             tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
@@ -173,7 +169,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             if tfa_results is False:
                 return False
 
-            if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', tfa_results) is not None:
+            if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
                 self._downloader.report_warning('Two-factor code expired. Please try again, or use a one-use backup code instead.')
                 return False
             if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:

From 201ea3ee8e392d6c82bb8137b80b4328db40a399 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 21:52:22 +0600
Subject: [PATCH 1072/2145] [extractor/common] Improve _hidden_inputs

---
 youtube_dl/extractor/common.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 16ae4b98f..e2ace827f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -724,16 +724,18 @@ class InfoExtractor(object):
 
     @staticmethod
     def _hidden_inputs(html):
-        return dict([
-            (input.group('name'), input.group('value')) for input in re.finditer(
-                r'''(?x)
-                    <input\s+
-                        type=(?P<q_hidden>["\'])hidden(?P=q_hidden)\s+
-                        name=(?P<q_name>["\'])(?P<name>.+?)(?P=q_name)\s+
-                        (?:id=(?P<q_id>["\']).+?(?P=q_id)\s+)?
-                        value=(?P<q_value>["\'])(?P<value>.*?)(?P=q_value)
-                ''', html)
-        ])
+        hidden_inputs = {}
+        for input in re.findall(r'<input([^>]+)>', html):
+            if not re.search(r'type=(["\'])hidden\1', input):
+                continue
+            name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
+            if not name:
+                continue
+            value = re.search(r'value=(["\'])(?P<value>.*?)\1', input)
+            if not value:
+                continue
+            hidden_inputs[name.group('value')] = value.group('value')
+        return hidden_inputs
 
     def _form_hidden_inputs(self, form_id, html):
         form = self._search_regex(

From e64b756943440d602dc757f81787cad6aee8f412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 21:55:07 +0600
Subject: [PATCH 1073/2145] [extractor/common] Interactive TFA code input

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e2ace827f..65835d257 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -15,6 +15,7 @@ import xml.etree.ElementTree
 from ..compat import (
     compat_cookiejar,
     compat_cookies,
+    compat_getpass,
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
@@ -610,7 +611,7 @@ class InfoExtractor(object):
 
         return (username, password)
 
-    def _get_tfa_info(self):
+    def _get_tfa_info(self, note='two-factor verification code'):
         """
         Get the two-factor authentication info
         TODO - asking the user will be required for sms/phone verify
@@ -624,7 +625,7 @@ class InfoExtractor(object):
         if downloader_params.get('twofactor', None) is not None:
             return downloader_params['twofactor']
 
-        return None
+        return compat_getpass('Type %s and press [Return]: ' % note)
 
     # Helper functions for extracting OpenGraph info
     @staticmethod

From 041bc3adc55bbe81649a1c5d283302e5a120659e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 22:03:43 +0600
Subject: [PATCH 1074/2145] [youtube] Simplify two-factor authentication

---
 youtube_dl/extractor/youtube.py | 35 +++++++++++----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bfa9a12a8..887c46d95 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,6 +33,7 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    remove_start,
     smuggle_url,
     str_to_int,
     unescapeHTML,
@@ -129,35 +130,23 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         # TODO add SMS and phone call support - these require making a request and then prompting the user
 
         if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
-            tfa_code = self._get_tfa_info()
+            tfa_code = self._get_tfa_info('2-step verification code')
 
-            if tfa_code is None:
-                self._downloader.report_warning('Two-factor authentication required. Provide it with --twofactor <code>')
-                self._downloader.report_warning('(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+            if not tfa_code:
+                self._downloader.report_warning(
+                    'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
+                    '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
                 return False
 
-            def find_value(element_id):
-                match = re.search(r'id="%s"\s+value="(.+?)">' % element_id, login_results, re.M | re.U)
-                if match is None:
-                    self._downloader.report_warning('Failed to get %s - did the page structure change?' % id)
-                return match.group(1)
+            tfa_code = remove_start(tfa_code, 'G-')
 
-            challengeId = find_value('challengeId')
-            challengeType = find_value('challengeType')
-            gxf = find_value('gxf')
+            tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
 
-            tfa_form_strs = {
-                'challengeId': challengeId,
-                'challengeType': challengeType,  # This doesn't appear to change
-                'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
-                'service': 'youtube',
-                'hl': 'en_US',
-                'checkedDomains': 'youtube',
-                'pstMsg': '0',
-                'gxf': gxf,
+            tfa_form_strs.update({
                 'Pin': tfa_code,
                 'TrustDevice': 'on',
-            }
+            })
+
             tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
             tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
 
@@ -170,7 +159,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 return False
 
             if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
-                self._downloader.report_warning('Two-factor code expired. Please try again, or use a one-use backup code instead.')
+                self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
                 return False
             if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
                 self._downloader.report_warning('unable to log in - did the page structure change?')

From eaa5646483d22d3b658dcf63b61e6c3b67aa5bc5 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 22:57:06 +0600
Subject: [PATCH 1075/2145] [README.md] Clarify configuration file usage
 (Closes #6530)

---
 README.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 542a7c26a..25844eb6d 100644
--- a/README.md
+++ b/README.md
@@ -236,7 +236,14 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, system wide configuration file is located at `/etc/youtube-dl.conf` and user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configration file youtube-dl will always extract the audio, not copy the mtime and use proxy:
+```
+--extract-audio
+--no-mtime
+--proxy 127.0.0.1:3128
+```
+
+You can use `--ignore-config` if you want to disable configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file ###
 

From 221a59fe6fe22ec286830319e72cbabdc83fd02f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Sun, 9 Aug 2015 12:27:31 +0200
Subject: [PATCH 1076/2145] [playtvak] Initial support for videos from Mafra
 group servers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Support for videos in articles from idnes.cz, lidovky.cz, metro.cz, as
well as video and live streams from playtvak.cz

Signed-off-by: Ondřej Caletka <ondrej@caletka.cz>
---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/playtvak.py | 127 +++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 youtube_dl/extractor/playtvak.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 83d21bd15..5307240f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -448,6 +448,7 @@ from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
+from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
new file mode 100644
index 000000000..6dff6650c
--- /dev/null
+++ b/youtube_dl/extractor/playtvak.py
@@ -0,0 +1,127 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse,
+)
+from ..utils import ExtractorError
+
+
+def _extract_json(code):
+    return re.sub(
+        r'(?s)^VideoPlayer.data\("", ({.*})\);?\s*?(?://[^\n]*)*$', r'\1', code)
+
+
+class PlaytvakIE(InfoExtractor):
+    _VALID_URL = r'https?://.*?(playtvak|idnes|lidovky|metro)\.cz/.*\?c=(?P<id>[A-Z][0-9]{6}_[0-9]{6}_.*)'
+    _TESTS = [{
+        'url': 'http://www.playtvak.cz/vyzente-vosy-a-srsne-ze-zahrady-dn5-/hodinovy-manzel.aspx?c=A150730_150323_hodinovy-manzel_kuko',
+        'md5': '4525ae312c324b4be2f4603cc78ceb4a',
+        'info_dict': {
+            'id': 'A150730_150323_hodinovy-manzel_kuko',
+            'ext': 'mp4',
+            'title': 'Vyžeňte vosy a sršně ze zahrady',
+            'thumbnail': 'http://oidnes.cz/15/074/mobil/KUK5cea00_010hodmanel58154.jpg',
+            'description': 'Málo co kazí atmosféru venkovního posezení tak jako neustálé bzučení kolem hlavy.  Vyzkoušejte náš lapač a odpuzovač vos a sršňů.',
+        }
+    }, {  # live video test
+        'url': 'http://slowtv.playtvak.cz/planespotting-0pr-/planespotting.aspx?c=A150624_164934_planespotting_cat',
+        'info_dict': {
+            'id': 'A150624_164934_planespotting_cat',
+            'ext': 'flv',
+            'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'thumbnail': 'http://data.idnes.cz/soubory/servisni-play-porady/89A150630_ACEK_026_VIDEOPLAYER-STREA.PNG',
+            'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }, {  # idnes.cz
+        'url': 'http://zpravy.idnes.cz/pes-zavreny-v-aute-rozbijeni-okynek-v-aute-fj5-/domaci.aspx?c=A150809_104116_domaci_pku',
+        'md5': '819832ba33cd7016e58a6658577fe289',
+        'info_dict': {
+            'id': 'A150809_104116_domaci_pku',
+            'ext': 'mp4',
+            'title': 'Zavřeli jsme mraženou pizzu do auta. Upekla se',
+            'thumbnail': 'http://i.idnes.cz/15/081/vidw/SHA5d1786_pizzaauto.jpg',
+            'description': 'Na sociálních sítích se objevila výzva, aby lidé, kteří v horkých letních dnech uvidí v zaparkovaném autě zavřeného psa, neváhali rozbít okénko. Zastánci tohoto postoje argumentují zdravím zvířete, které v dusnu může zkolabovat. Policie doporučuje nejprve volat tísňovou linku.',
+        }
+    }, {  # lidovky.cz
+        'url': 'http://www.lidovky.cz/dalsi-demonstrace-v-praze-o-migraci-duq-/video.aspx?c=A150808_214044_ln-video_ELE',
+        'md5': 'c7209ac4ba9d234d4ad5bab7485bcee8',
+        'info_dict': {
+            'id': 'A150808_214044_ln-video_ELE',
+            'ext': 'mp4',
+            'title': 'Táhni! Demonstrace proti imigrantům budila emoce',
+            'thumbnail': 'http://i.idnes.cz/15/081/vidw/PID5d1d52_vandas3.jpg',
+            'description': 'Desítky lidí se sešly v Praze na protest proti imigrantům. Současně probíhala i demonstrace na jejich podporu. Na Staroměstském náměstí vystoupil i předseda dělnické strany Tomáš Vandas a kontroverzní slovenský politik Marian Kotleba. Dalšího slovenského nacionalistu Mariána Magáta odvedla policie.',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        infourl = self._html_search_regex(r'Misc.videoFLV\({ data: "([^"]+)"', webpage, 'xmlinfourl')
+        parsedurl = compat_urlparse.urlparse(infourl)
+        qs = compat_urlparse.parse_qs(parsedurl.query)
+        if 'reklama' in qs:  # Don't ask for ads
+            qs['reklama'] = ['0']
+        qs['type'] = ['js']  # Ask for JS-based info file
+        newquery = compat_urllib_parse.urlencode(qs, True)
+        infourl = compat_urlparse.urlunparse(parsedurl[:4] + (newquery, ''))
+        jsoninfo = self._download_json(infourl, video_id, transform_source=_extract_json)
+
+        item = None
+        for i in jsoninfo['items']:
+            if i['type'] == 'video' or i['type'] == 'stream':
+                item = i
+                break
+        if item is None:
+            raise ExtractorError('No suitable stream found')
+        title = item['title']
+        thumbnail = item['image']
+        is_live = item['type'] == 'stream'
+        if is_live:
+            title = self._live_title(title)
+
+        formats = []
+        for fmt in item['video']:
+            format_entry = {'url': fmt['file'],
+                            'format_id': ("%s_%s" % (fmt['format'], fmt['quality'])),
+                            }
+            if fmt['quality'] == 'middle':
+                format_entry['quality'] = -2
+            elif fmt['quality'] == 'low':
+                format_entry['quality'] = -3
+
+            if fmt['format'] == 'mp4':
+                format_entry['ext'] = 'mp4'
+            elif fmt['format'] == 'webm':
+                format_entry['ext'] = 'webm'
+            elif fmt['format'] == 'apple':
+                format_entry['ext'] = 'mp4'
+                format_entry['protocol'] = 'm3u8'
+                # Some streams have mp3 audio which does not play
+                # well with ffmpeg filter aac_adtstoasc
+                format_entry['preference'] = -1
+            elif fmt['format'] == 'rtmp':
+                format_entry['ext'] = 'flv'
+            else:  # Other formats not supported yet
+                continue
+
+            formats.append(format_entry)
+
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'description': self._og_search_description(webpage),
+            'is_live': is_live,
+            'formats': formats,
+        }

From 276c9897720fe087924aef4ac80cf528e621b832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 03:07:05 +0600
Subject: [PATCH 1077/2145] [playtvak] Improve and simplify

---
 youtube_dl/extractor/playtvak.py | 145 ++++++++++++++++++++-----------
 1 file changed, 92 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 6dff6650c..4e5034dc6 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -1,23 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
 )
-from ..utils import ExtractorError
-
-
-def _extract_json(code):
-    return re.sub(
-        r'(?s)^VideoPlayer.data\("", ({.*})\);?\s*?(?://[^\n]*)*$', r'\1', code)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
+    qualities,
+)
 
 
 class PlaytvakIE(InfoExtractor):
-    _VALID_URL = r'https?://.*?(playtvak|idnes|lidovky|metro)\.cz/.*\?c=(?P<id>[A-Z][0-9]{6}_[0-9]{6}_.*)'
+    IE_DESC = 'Playtvak.cz, iDNES.cz and Lidovky.cz'
+    _VALID_URL = r'https?://(?:.+?\.)?(?:playtvak|idnes|lidovky|metro)\.cz/.*\?(?:c|idvideo)=(?P<id>[^&]+)'
     _TESTS = [{
         'url': 'http://www.playtvak.cz/vyzente-vosy-a-srsne-ze-zahrady-dn5-/hodinovy-manzel.aspx?c=A150730_150323_hodinovy-manzel_kuko',
         'md5': '4525ae312c324b4be2f4603cc78ceb4a',
@@ -25,8 +24,12 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150730_150323_hodinovy-manzel_kuko',
             'ext': 'mp4',
             'title': 'Vyžeňte vosy a sršně ze zahrady',
-            'thumbnail': 'http://oidnes.cz/15/074/mobil/KUK5cea00_010hodmanel58154.jpg',
-            'description': 'Málo co kazí atmosféru venkovního posezení tak jako neustálé bzučení kolem hlavy.  Vyzkoušejte náš lapač a odpuzovač vos a sršňů.',
+            'description': 'md5:f93d398691044d303bc4a3de62f3e976',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'duration': 279,
+            'timestamp': 1438732860,
+            'upload_date': '20150805',
+            'is_live': False,
         }
     }, {  # live video test
         'url': 'http://slowtv.playtvak.cz/planespotting-0pr-/planespotting.aspx?c=A150624_164934_planespotting_cat',
@@ -34,8 +37,8 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150624_164934_planespotting_cat',
             'ext': 'flv',
             'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'thumbnail': 'http://data.idnes.cz/soubory/servisni-play-porady/89A150630_ACEK_026_VIDEOPLAYER-STREA.PNG',
             'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
             'is_live': True,
         },
         'params': {
@@ -48,8 +51,12 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150809_104116_domaci_pku',
             'ext': 'mp4',
             'title': 'Zavřeli jsme mraženou pizzu do auta. Upekla se',
-            'thumbnail': 'http://i.idnes.cz/15/081/vidw/SHA5d1786_pizzaauto.jpg',
-            'description': 'Na sociálních sítích se objevila výzva, aby lidé, kteří v horkých letních dnech uvidí v zaparkovaném autě zavřeného psa, neváhali rozbít okénko. Zastánci tohoto postoje argumentují zdravím zvířete, které v dusnu může zkolabovat. Policie doporučuje nejprve volat tísňovou linku.',
+            'description': 'md5:01e73f02329e2e5760bd5eed4d42e3c2',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'duration': 39,
+            'timestamp': 1438969140,
+            'upload_date': '20150807',
+            'is_live': False,
         }
     }, {  # lidovky.cz
         'url': 'http://www.lidovky.cz/dalsi-demonstrace-v-praze-o-migraci-duq-/video.aspx?c=A150808_214044_ln-video_ELE',
@@ -58,70 +65,102 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150808_214044_ln-video_ELE',
             'ext': 'mp4',
             'title': 'Táhni! Demonstrace proti imigrantům budila emoce',
-            'thumbnail': 'http://i.idnes.cz/15/081/vidw/PID5d1d52_vandas3.jpg',
-            'description': 'Desítky lidí se sešly v Praze na protest proti imigrantům. Současně probíhala i demonstrace na jejich podporu. Na Staroměstském náměstí vystoupil i předseda dělnické strany Tomáš Vandas a kontroverzní slovenský politik Marian Kotleba. Dalšího slovenského nacionalistu Mariána Magáta odvedla policie.',
+            'description': 'md5:97c81d589a9491fbfa323c9fa3cca72c',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1439052180,
+            'upload_date': '20150808',
+            'is_live': False,
         }
+    }, {
+        'url': 'http://www.playtvak.cz/embed.aspx?idvideo=V150729_141549_play-porad_kuko',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        infourl = self._html_search_regex(r'Misc.videoFLV\({ data: "([^"]+)"', webpage, 'xmlinfourl')
-        parsedurl = compat_urlparse.urlparse(infourl)
-        qs = compat_urlparse.parse_qs(parsedurl.query)
-        if 'reklama' in qs:  # Don't ask for ads
-            qs['reklama'] = ['0']
-        qs['type'] = ['js']  # Ask for JS-based info file
-        newquery = compat_urllib_parse.urlencode(qs, True)
-        infourl = compat_urlparse.urlunparse(parsedurl[:4] + (newquery, ''))
-        jsoninfo = self._download_json(infourl, video_id, transform_source=_extract_json)
+
+        info_url = self._html_search_regex(
+            r'Misc\.videoFLV\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')
+
+        parsed_url = compat_urlparse.urlparse(info_url)
+
+        qs = compat_urlparse.parse_qs(parsed_url.query)
+        qs.update({
+            'reklama': ['0'],
+            'type': ['js'],
+        })
+
+        info_url = compat_urlparse.urlunparse(
+            parsed_url._replace(query = compat_urllib_parse.urlencode(qs, True)))
+
+        json_info = self._download_json(
+            info_url, video_id,
+            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
 
         item = None
-        for i in jsoninfo['items']:
-            if i['type'] == 'video' or i['type'] == 'stream':
+        for i in json_info['items']:
+            if i.get('type') == 'video' or i.get('type') == 'stream':
                 item = i
                 break
-        if item is None:
+        if not item:
             raise ExtractorError('No suitable stream found')
-        title = item['title']
-        thumbnail = item['image']
-        is_live = item['type'] == 'stream'
-        if is_live:
-            title = self._live_title(title)
+
+        quality = qualities(['low', 'middle', 'high'])
 
         formats = []
         for fmt in item['video']:
-            format_entry = {'url': fmt['file'],
-                            'format_id': ("%s_%s" % (fmt['format'], fmt['quality'])),
-                            }
-            if fmt['quality'] == 'middle':
-                format_entry['quality'] = -2
-            elif fmt['quality'] == 'low':
-                format_entry['quality'] = -3
+            video_url = fmt.get('file')
+            if not video_url:
+                continue
 
-            if fmt['format'] == 'mp4':
-                format_entry['ext'] = 'mp4'
-            elif fmt['format'] == 'webm':
-                format_entry['ext'] = 'webm'
-            elif fmt['format'] == 'apple':
-                format_entry['ext'] = 'mp4'
-                format_entry['protocol'] = 'm3u8'
+            format_ = fmt['format']
+            format_id = '%s_%s' % (format_, fmt['quality'])
+            preference = None
+
+            if format_ in ['mp4', 'webm']:
+                ext = format_
+            elif format_ == 'rtmp':
+                ext = 'flv'
+            elif format_ == 'apple':
+                ext = 'mp4'
                 # Some streams have mp3 audio which does not play
                 # well with ffmpeg filter aac_adtstoasc
-                format_entry['preference'] = -1
-            elif fmt['format'] == 'rtmp':
-                format_entry['ext'] = 'flv'
+                preference = -1
+            elif format_ == 'adobe':  # f4m manifest fails with 404 in 80% of requests
+                continue
             else:  # Other formats not supported yet
                 continue
 
-            formats.append(format_entry)
-
+            formats.append({
+                'url': video_url,
+                'ext': ext,
+                'format_id': format_id,
+                'quality': quality(fmt.get('quality')),
+                'preference': preference,
+            })
         self._sort_formats(formats)
+
+        title = item['title']
+        is_live = item['type'] == 'stream'
+        if is_live:
+            title = self._live_title(title)
+        timestamp = None
+        duration = None
+        if not is_live:
+            duration = int_or_none(item.get('length'))
+            timestamp = item.get('published')
+            if timestamp:
+                timestamp = parse_iso8601(timestamp[:-5])
+
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
+            'thumbnail': item.get('image'),
+            'duration': duration,
+            'timestamp': timestamp,
             'is_live': is_live,
             'formats': formats,
         }

From 6900b4f6f52ba783bb7a6028fd174250c8832a38 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Aug 2015 01:05:04 +0200
Subject: [PATCH 1078/2145] release 2015.08.16

---
 CONTRIBUTING.md        | 2 +-
 docs/supportedsites.md | 6 ++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 588b15bde..42333c450 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -125,7 +125,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e21471102..9099e2da4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -150,6 +150,7 @@
  - **EroProfile**
  - **Escapist**
  - **ESPN** (Currently broken)
+ - **EsriVideo**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -220,6 +221,8 @@
  - **imdb:list**: Internet Movie Database lists
  - **Imgur**
  - **Ina**
+ - **Indavideo**
+ - **IndavideoEmbed**
  - **InfoQ**
  - **Instagram**
  - **instagram:user**: Instagram user profile
@@ -386,6 +389,7 @@
  - **PlanetaPlay**
  - **play.fm**
  - **played.to**
+ - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **Playwire**
  - **plus.google**: Google Plus
@@ -432,6 +436,7 @@
  - **rtve.es:alacarta**: RTVE a la carta
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
+ - **RTVNH**
  - **RUHD**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
@@ -455,6 +460,7 @@
  - **ServingSys**
  - **Sexu**
  - **SexyKarma**: Sexy Karma and Watch Indian Porn
+ - **Shahid**
  - **Shared**
  - **ShareSix**
  - **Sina**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6462d4477..689d6fca7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.09'
+__version__ = '2015.08.16'

From 974f1a385a452b1c86d6f3ff16035b30baaeeb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 17:22:13 +0600
Subject: [PATCH 1079/2145] [playtvak] Improve description extraction and add
 test for metro

---
 youtube_dl/extractor/playtvak.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 4e5034dc6..2b338966f 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -71,6 +71,19 @@ class PlaytvakIE(InfoExtractor):
             'upload_date': '20150808',
             'is_live': False,
         }
+    }, {  # metro.cz
+        'url': 'http://www.metro.cz/video-pod-billboardem-se-na-vltavske-roztocil-kolotoc-deti-vozil-jen-par-hodin-1hx-/metro-extra.aspx?c=A141111_173251_metro-extra_row',
+        'md5': '84fc1deedcac37b7d4a6ccae7c716668',
+        'info_dict': {
+            'id': 'A141111_173251_metro-extra_row',
+            'ext': 'mp4',
+            'title': 'Recesisté udělali z billboardu kolotoč',
+            'description': 'md5:7369926049588c3989a66c9c1a043c4c',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1415725500,
+            'upload_date': '20141111',
+            'is_live': False,
+        }
     }, {
         'url': 'http://www.playtvak.cz/embed.aspx?idvideo=V150729_141549_play-porad_kuko',
         'only_matching': True,
@@ -146,6 +159,8 @@ class PlaytvakIE(InfoExtractor):
         is_live = item['type'] == 'stream'
         if is_live:
             title = self._live_title(title)
+        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
+            'description', webpage, 'description')
         timestamp = None
         duration = None
         if not is_live:
@@ -157,7 +172,7 @@ class PlaytvakIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': self._og_search_description(webpage),
+            'description': description,
             'thumbnail': item.get('image'),
             'duration': duration,
             'timestamp': timestamp,

From 7fc18d930917ad407c78bb9b0465dc4fae2fb335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 19:53:14 +0600
Subject: [PATCH 1080/2145] [screenwavemedia] Fix extraction (Closes #6575)

---
 youtube_dl/extractor/screenwavemedia.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 3bc84989e..78b068be2 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -35,15 +37,18 @@ class ScreenwaveMediaIE(InfoExtractor):
 
         sources = self._parse_json(
             js_to_json(
-                self._search_regex(
-                    r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
-                    'sources',
-                ).replace(
-                    "' + thisObj.options.videoserver + '",
-                    videoserver
-                ).replace(
-                    "' + playerVidId + '",
-                    video_id
+                re.sub(
+                    r'(?s)/\*.*?\*/', '',
+                    self._search_regex(
+                        r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                        'sources',
+                    ).replace(
+                        "' + thisObj.options.videoserver + '",
+                        videoserver
+                    ).replace(
+                        "' + playerVidId + '",
+                        video_id
+                    )
                 )
             ),
             video_id

From 008687427725b8d857c44d75f358059c2533539a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 19:59:03 +0600
Subject: [PATCH 1081/2145] [playtvak] Use tuples

---
 youtube_dl/extractor/playtvak.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 2b338966f..278fdc1aa 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -120,7 +120,7 @@ class PlaytvakIE(InfoExtractor):
         if not item:
             raise ExtractorError('No suitable stream found')
 
-        quality = qualities(['low', 'middle', 'high'])
+        quality = qualities(('low', 'middle', 'high'))
 
         formats = []
         for fmt in item['video']:
@@ -132,7 +132,7 @@ class PlaytvakIE(InfoExtractor):
             format_id = '%s_%s' % (format_, fmt['quality'])
             preference = None
 
-            if format_ in ['mp4', 'webm']:
+            if format_ in ('mp4', 'webm'):
                 ext = format_
             elif format_ == 'rtmp':
                 ext = 'flv'

From 8626b23e4ea091c4093c25626ca9fc12293b2830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 20:18:15 +0600
Subject: [PATCH 1082/2145] [screenwavemedia] Make more robust

---
 youtube_dl/extractor/screenwavemedia.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 78b068be2..220d39078 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -51,19 +51,38 @@ class ScreenwaveMediaIE(InfoExtractor):
                     )
                 )
             ),
-            video_id
+            video_id, fatal=False
         )
 
+        # Fallback to hardcoded sources if JS changes again
+        if not sources:
+            sources = [{
+                'file': 'http://%s/vod/%s_%s.mp4' % (videoserver, video_id, format_id),
+                'type': 'mp4',
+                'label': format_label,
+            } for format_id, format_label in (
+                ('low', '144p Low'), ('med', '160p Med'), ('high', '360p High'), ('hd1', '720p HD1'))]
+            sources.append({
+                'file': 'http://%s/vod/smil:%s.smil/playlist.m3u8' % (videoserver, video_id),
+                'type': 'hls',
+            })
+
         formats = []
         for source in sources:
             if source['type'] == 'hls':
                 formats.extend(self._extract_m3u8_formats(source['file'], video_id))
             else:
+                file_ = source.get('file')
+                if not file_:
+                    continue
                 format_label = source.get('label')
+                format_id = self._search_regex(
+                    r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]', format_label, 'height', default=None))
                 formats.append({
                     'url': source['file'],
+                    'format_id': format_id,
                     'format': format_label,
                     'ext': source.get('type'),
                     'height': height,

From f74a7348f6ac52259ea66b74a40165b448fbd702 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 23:33:17 +0600
Subject: [PATCH 1083/2145] [youtube:search_url] Fix extraction (Closes #6578)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 887c46d95..4d1ca9298 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1762,7 +1762,7 @@ class YoutubeSearchURLIE(InfoExtractor):
             r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
 
         part_codes = re.findall(
-            r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)
+            r'(?s)<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*>(.*?)</h3>', result_code)
         entries = []
         for part_code in part_codes:
             part_title = self._html_search_regex(

From cbaed4bb5e5e90103a1164d9326043a3abd0bf83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 02:04:13 +0600
Subject: [PATCH 1084/2145] [youtube] Expand _VALID_URL to support vid.plus

---
 youtube_dl/extractor/youtube.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4d1ca9298..8e2da46e3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -202,7 +202,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                  v=
                              )
                          ))
-                         |youtu\.be/                                          # just youtu.be/xxxx
+                         |(?:
+                            youtu\.be|                                        # just youtu.be/xxxx
+                            vid\.plus                                         # or vid.plus/xxxx
+                         )/
                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
                          )
                      )?                                                       # all until now is optional -> you can pass the naked ID
@@ -624,6 +627,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+        },
+        {
+            'url': 'http://vid.plus/FlRa-iH7PGw',
+            'only_matching': True,
         }
     ]
 

From c00c7c0af0fdcb380aef0ea9e072a61979d17816 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Aug 2015 23:39:50 +0200
Subject: [PATCH 1085/2145] [sportdeutschland] Fix extraction

---
 youtube_dl/extractor/sportdeutschland.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 1a57aebf1..7ec6c613f 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -38,10 +38,12 @@ class SportDeutschlandIE(InfoExtractor):
             'upload_date': '20140825',
             'description': 'md5:60a20536b57cee7d9a4ec005e8687504',
             'timestamp': 1408976060,
+            'duration': 2732,
             'title': 'Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen: Herren Einzel, Wei Lee vs. Keun Lee',
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'categories': ['Li-Ning Badminton WM 2014'],
+
         }
     }]
 
@@ -50,7 +52,7 @@ class SportDeutschlandIE(InfoExtractor):
         video_id = mobj.group('id')
         sport_id = mobj.group('sport')
 
-        api_url = 'http://splink.tv/api/permalinks/%s/%s' % (
+        api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
             sport_id, video_id)
         req = compat_urllib_request.Request(api_url, headers={
             'Accept': 'application/vnd.vidibus.v2.html+json',
@@ -58,12 +60,11 @@ class SportDeutschlandIE(InfoExtractor):
         })
         data = self._download_json(req, video_id)
 
-        categories = list(data.get('section', {}).get('tags', {}).values())
         asset = data['asset']
-        assets_info = self._download_json(asset['url'], video_id)
+        categories = [data['section']['title']]
 
         formats = []
-        smil_url = assets_info['video']
+        smil_url = asset['video']
         if '.smil' in smil_url:
             m3u8_url = smil_url.replace('.smil', '.m3u8')
             formats.extend(
@@ -91,6 +92,7 @@ class SportDeutschlandIE(InfoExtractor):
             'title': asset['title'],
             'thumbnail': asset.get('image'),
             'description': asset.get('teaser'),
+            'duration': asset.get('duration'),
             'categories': categories,
             'view_count': asset.get('views'),
             'rtmp_live': asset.get('live'),

From 0fa5795b85f8d97bf67f10e39a79b49656be58db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Aug 2015 23:40:07 +0200
Subject: [PATCH 1086/2145] release 2015.08.16.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 689d6fca7..c090c6df7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.16'
+__version__ = '2015.08.16.1'

From 369c12e038c3183a0e725a929dd9bed4ec35fa11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 20:16:43 +0600
Subject: [PATCH 1087/2145] [twitch] Allow untitled videos (Closes #6585)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a2b6a35aa..0521257e5 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -132,7 +132,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
     def _extract_info(self, info):
         return {
             'id': info['_id'],
-            'title': info['title'],
+            'title': info.get('title') or 'Untitled Broadcast',
             'description': info['description'],
             'duration': info['length'],
             'thumbnail': info['preview'],

From 7a6e8a1b17a6a821d9200531ebf65562ccc2d428 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 20:20:04 +0600
Subject: [PATCH 1088/2145] [twitch] Make more robust

---
 youtube_dl/extractor/twitch.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 0521257e5..8cba97bd4 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,6 +15,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     parse_duration,
     parse_iso8601,
 )
@@ -133,13 +134,13 @@ class TwitchItemBaseIE(TwitchBaseIE):
         return {
             'id': info['_id'],
             'title': info.get('title') or 'Untitled Broadcast',
-            'description': info['description'],
-            'duration': info['length'],
-            'thumbnail': info['preview'],
-            'uploader': info['channel']['display_name'],
-            'uploader_id': info['channel']['name'],
-            'timestamp': parse_iso8601(info['recorded_at']),
-            'view_count': info['views'],
+            'description': info.get('description'),
+            'duration': int_or_none(info.get('length')),
+            'thumbnail': info.get('preview'),
+            'uploader': info.get('channel', {}).get('display_name'),
+            'uploader_id': info.get('channel', {}).get('name'),
+            'timestamp': parse_iso8601(info.get('recorded_at')),
+            'view_count': int_or_none(info.get('views')),
         }
 
     def _real_extract(self, url):

From 9c724a980210ec6a7659fe869cce401dde6e189d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 20:23:52 +0600
Subject: [PATCH 1089/2145] [twitch:vod] Add test for #6585

---
 youtube_dl/extractor/twitch.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8cba97bd4..4f4eb6d72 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -188,7 +188,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
         'info_dict': {
             'id': 'v6528877',
@@ -207,7 +207,26 @@ class TwitchVodIE(TwitchItemBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        # Untitled broadcast (title is None)
+        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
+        'info_dict': {
+            'id': 'v11230755',
+            'ext': 'mp4',
+            'title': 'Untitled Broadcast',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1638,
+            'timestamp': 1439746708,
+            'upload_date': '20150816',
+            'uploader': 'BelkAO_o',
+            'uploader_id': 'belkao_o',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         item_id = self._match_id(url)

From 3b9b32f404ab09d9dc801dd8ec57d79711be5cb3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 18 Aug 2015 13:02:41 +0200
Subject: [PATCH 1090/2145] [libsyn] Strip options from player URL

---
 youtube_dl/extractor/libsyn.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 9ab1416f5..d375695f5 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -8,9 +8,9 @@ from ..utils import unified_strdate
 
 
 class LibsynIE(InfoExtractor):
-    _VALID_URL = r'https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)'
+    _VALID_URL = r'(?P<mainurl>https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+))'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://html5-player.libsyn.com/embed/episode/id/3377616/',
         'md5': '443360ee1b58007bc3dcf09b41d093bb',
         'info_dict': {
@@ -19,12 +19,24 @@ class LibsynIE(InfoExtractor):
             'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
             'description': 'md5:601cb790edd05908957dae8aaa866465',
             'upload_date': '20150220',
+            'thumbnail': 're:^https?://.*',
         },
-    }
+    }, {
+        'url': 'https://html5-player.libsyn.com/embed/episode/id/3727166/height/75/width/200/theme/standard/direction/no/autoplay/no/autonext/no/thumbnail/no/preload/no/no_addthis/no/',
+        'md5': '6c5cb21acd622d754d3b1a92b582ce42',
+        'info_dict': {
+            'id': '3727166',
+            'ext': 'mp3',
+            'title': 'Clients From Hell Podcast - How a Sex Toy Company Kickstarted my Freelance Career',
+            'upload_date': '20150818',
+            'thumbnail': 're:^https?://.*',
+        }
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+        url = m.group('mainurl')
         webpage = self._download_webpage(url, video_id)
 
         formats = [{
@@ -32,20 +44,18 @@ class LibsynIE(InfoExtractor):
         } for media_url in set(re.findall('var\s+mediaURL(?:Libsyn)?\s*=\s*"([^"]+)"', webpage))]
 
         podcast_title = self._search_regex(
-            r'<h2>([^<]+)</h2>', webpage, 'title')
+            r'<h2>([^<]+)</h2>', webpage, 'podcast title', default=None)
         episode_title = self._search_regex(
-            r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
+            r'(?:<div class="episode-title">|<h3>)([^<]+)</', webpage, 'episode title')
 
         title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title
 
         description = self._html_search_regex(
             r'<div id="info_text_body">(.+?)</div>', webpage,
-            'description', fatal=False)
-
+            'description', default=None)
         thumbnail = self._search_regex(
             r'<img[^>]+class="info-show-icon"[^>]+src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
-
         release_date = unified_strdate(self._search_regex(
             r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))
 

From 34a4cd0a34bc9f07d865b02f6982fba60421ed0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 20:02:56 +0600
Subject: [PATCH 1091/2145] [telecinco] Relax _VALID_URL (Closes #6601)

---
 youtube_dl/extractor/telecinco.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index a0c744fd1..ae94f055c 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,7 +6,7 @@ from .mitele import MiTeleIE
 
 class TelecincoIE(MiTeleIE):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/(?:[^/]+/)?(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
@@ -23,4 +23,7 @@ class TelecincoIE(MiTeleIE):
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
+        'only_matching': True,
     }]

From 03c635a4b57e6ea4b874029d9fe3738508f6fc7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 20:26:45 +0600
Subject: [PATCH 1092/2145] [twitch] Fix login (Closes #6599)

---
 youtube_dl/extractor/twitch.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4f4eb6d72..023911c41 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -12,6 +12,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -27,7 +28,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/login'
-    _LOGIN_POST_URL = 'https://passport.twitch.tv/authorize'
+    _LOGIN_POST_URL = 'https://passport.twitch.tv/authentications/new'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -70,8 +71,15 @@ class TwitchBaseIE(InfoExtractor):
             'password': password.encode('utf-8'),
         })
 
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_POST_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
         request = compat_urllib_request.Request(
-            self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)

From 559f4c550f215a657ce386cab572bfc212128595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 20:27:58 +0600
Subject: [PATCH 1093/2145] [playtvak] PEP 8

---
 youtube_dl/extractor/playtvak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 278fdc1aa..e360404f7 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -106,7 +106,7 @@ class PlaytvakIE(InfoExtractor):
         })
 
         info_url = compat_urlparse.urlunparse(
-            parsed_url._replace(query = compat_urllib_parse.urlencode(qs, True)))
+            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
 
         json_info = self._download_json(
             info_url, video_id,

From f3a65d9636908ee49ff3d50c24efb8067caa32c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 21:10:52 +0600
Subject: [PATCH 1094/2145] [travis] Move to new infrastructure

We don't use rtmpdump in tests anyway
---
 .travis.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 511bee64c..e78a2fa76 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,9 +5,7 @@ python:
   - "3.2"
   - "3.3"
   - "3.4"
-before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -yqq rtmpdump
+sudo: false
 script: nosetests test --verbose
 notifications:
   email:

From a01da8bbf83dfd4f87e3fdd105b9f7c850e76cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 23:02:57 +0600
Subject: [PATCH 1095/2145] [crunchyroll] Workaround fplive.net rtmp URLs
 (Closes #5881)

---
 youtube_dl/extractor/crunchyroll.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d1b6d7366..33a033a7f 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -14,11 +14,13 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
     intlist_to_bytes,
+    remove_end,
     unified_strdate,
     urlencode_postdata,
 )
@@ -279,6 +281,20 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             stream_info = streamdata.find('./{default}preload/stream_info')
             video_url = stream_info.find('./host').text
             video_play_path = stream_info.find('./file').text
+
+            if '.fplive.net/' in video_url:
+                video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
+                parsed_video_url = compat_urlparse.urlparse(video_url)
+                direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
+                    netloc='v.lvlt.crcdn.net',
+                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
+                if self._is_valid_url(direct_video_url, video_id, video_format):
+                    formats.append({
+                        'url': direct_video_url,
+                        'format_id': video_format,
+                    })
+                    continue
+
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From ca681f7041838fa215f8ab5266cd7b442f3f9445 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Aug 2015 20:52:36 +0600
Subject: [PATCH 1096/2145] [videobam] Remove extractor

videobam.com redirects to sendvid.com now
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/videobam.py | 81 --------------------------------
 2 files changed, 82 deletions(-)
 delete mode 100644 youtube_dl/extractor/videobam.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fa9acc923..c8c9f1855 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -696,7 +696,6 @@ from .vgtv import (
 from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
-from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
diff --git a/youtube_dl/extractor/videobam.py b/youtube_dl/extractor/videobam.py
deleted file mode 100644
index 0eb3d9414..000000000
--- a/youtube_dl/extractor/videobam.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-import json
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class VideoBamIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videobam\.com/(?:videos/download/)?(?P<id>[a-zA-Z]+)'
-
-    _TESTS = [
-        {
-            'url': 'http://videobam.com/OiJQM',
-            'md5': 'db471f27763a531f10416a0c58b5a1e0',
-            'info_dict': {
-                'id': 'OiJQM',
-                'ext': 'mp4',
-                'title': 'Is Alcohol Worse Than Ecstasy?',
-                'description': 'md5:d25b96151515c91debc42bfbb3eb2683',
-                'uploader': 'frihetsvinge',
-            },
-        },
-        {
-            'url': 'http://videobam.com/pqLvq',
-            'md5': 'd9a565b5379a99126ef94e1d7f9a383e',
-            'note': 'HD video',
-            'info_dict': {
-                'id': 'pqLvq',
-                'ext': 'mp4',
-                'title': '_',
-            }
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage('http://videobam.com/%s' % video_id, video_id, 'Downloading page')
-
-        formats = []
-
-        for preference, format_id in enumerate(['low', 'high']):
-            mobj = re.search(r"%s: '(?P<url>[^']+)'" % format_id, page)
-            if not mobj:
-                continue
-            formats.append({
-                'url': mobj.group('url'),
-                'ext': 'mp4',
-                'format_id': format_id,
-                'preference': preference,
-            })
-
-        if not formats:
-            player_config = json.loads(self._html_search_regex(r'var player_config = ({.+?});', page, 'player config'))
-            formats = [{
-                'url': item['url'],
-                'ext': 'mp4',
-            } for item in player_config['playlist'] if 'autoPlay' in item]
-
-        self._sort_formats(formats)
-
-        title = self._og_search_title(page, default='_', fatal=False)
-        description = self._og_search_description(page, default=None)
-        thumbnail = self._og_search_thumbnail(page)
-        uploader = self._html_search_regex(r'Upload by ([^<]+)</a>', page, 'uploader', fatal=False, default=None)
-        view_count = int_or_none(
-            self._html_search_regex(r'<strong>Views:</strong> (\d+) ', page, 'view count', fatal=False))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'view_count': view_count,
-            'formats': formats,
-            'age_limit': 18,
-        }

From f877c6ae5a6e252d6904f90d597479451d2107aa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 19 Aug 2015 23:11:25 +0800
Subject: [PATCH 1097/2145] [theplatform] Use
 InfoExtractor._parse_smil_formats()

---
 youtube_dl/extractor/common.py      |  8 +++-
 youtube_dl/extractor/theplatform.py | 72 ++++++-----------------------
 2 files changed, 20 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 65835d257..ac12be933 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1052,7 +1052,7 @@ class InfoExtractor(object):
         return self._search_regex(
             r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
 
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         base = smil_url
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
             b = meta.get('base') or meta.get('httpBase')
@@ -1091,6 +1091,12 @@ class InfoExtractor(object):
                     'width': width,
                     'height': height,
                 })
+                if transform_rtmp_url:
+                    streamer, src = transform_rtmp_url(streamer, src)
+                    formats[-1].update({
+                        'url': streamer,
+                        'play_path': src,
+                    })
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0643eccaf..29f938a76 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -9,9 +9,6 @@ import hashlib
 
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -20,7 +17,8 @@ from ..utils import (
     int_or_none,
 )
 
-_x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
+default_ns = 'http://www.w3.org/2005/SMIL21/Language'
+_x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
 class ThePlatformIE(InfoExtractor):
@@ -145,63 +143,19 @@ class ThePlatformIE(InfoExtractor):
                     'url': src,
                 }]
 
-        head = meta.find(_x('smil:head'))
-        body = meta.find(_x('smil:body'))
-
-        f4m_node = body.find(_x('smil:seq//smil:video'))
-        if f4m_node is None:
-            f4m_node = body.find(_x('smil:seq/smil:video'))
-        if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
-            f4m_url = f4m_node.attrib['src']
-            if 'manifest.f4m?' not in f4m_url:
-                f4m_url += '?'
+        formats = self._parse_smil_formats(
+            meta, smil_url, video_id, namespace=default_ns,
             # the parameters are from syfy.com, other sites may use others,
             # they also work for nbc.com
-            f4m_url += '&g=UXWGVKRWHFSP&hdcore=3.0.3'
-            formats = self._extract_f4m_formats(f4m_url, video_id)
-        else:
-            formats = []
-            switch = body.find(_x('smil:switch'))
-            if switch is None:
-                switch = body.find(_x('smil:par//smil:switch'))
-            if switch is None:
-                switch = body.find(_x('smil:par/smil:switch'))
-            if switch is None:
-                switch = body.find(_x('smil:par'))
-            if switch is not None:
-                base_url = head.find(_x('smil:meta')).attrib['base']
-                for f in switch.findall(_x('smil:video')):
-                    attr = f.attrib
-                    width = int_or_none(attr.get('width'))
-                    height = int_or_none(attr.get('height'))
-                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
-                    format_id = '%dx%d_%dk' % (width, height, vbr)
-                    formats.append({
-                        'format_id': format_id,
-                        'url': base_url,
-                        'play_path': 'mp4:' + attr['src'],
-                        'ext': 'flv',
-                        'width': width,
-                        'height': height,
-                        'vbr': vbr,
-                    })
-            else:
-                switch = body.find(_x('smil:seq//smil:switch'))
-                if switch is None:
-                    switch = body.find(_x('smil:seq/smil:switch'))
-                for f in switch.findall(_x('smil:video')):
-                    attr = f.attrib
-                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
-                    ext = determine_ext(attr['src'])
-                    if ext == 'once':
-                        ext = 'mp4'
-                    formats.append({
-                        'format_id': compat_str(vbr),
-                        'url': attr['src'],
-                        'vbr': vbr,
-                        'ext': ext,
-                    })
-            self._sort_formats(formats)
+            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
+            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
+
+        for _format in formats:
+            ext = determine_ext(_format['url'])
+            if ext == 'once':
+                _format['ext'] = 'mp4'
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 7900aede14d2e2a46c8fd4430e48cde41f354859 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 22 Jul 2015 14:31:29 +0800
Subject: [PATCH 1098/2145] [mwave] New extractor for mwave.interest.me

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mwave.py    | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/mwave.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c8c9f1855..006ef3922 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -345,6 +345,7 @@ from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
 from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
+from .mwave import MwaveIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvi import MyviIE
diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
new file mode 100644
index 000000000..7f91aa269
--- /dev/null
+++ b/youtube_dl/extractor/mwave.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MwaveIE(InfoExtractor):
+    IE_NAME = 'mwave'
+    _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
+        'info_dict': {
+            'id': '168859',
+            'ext': 'flv',
+            'title': '[M COUNTDOWN] SISTAR - SHAKE IT',
+            'creator': 'M COUNTDOWN',
+        }
+    }, {
+        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168860',
+        'info_dict': {
+            'id': '168860',
+            'ext': 'flv',
+            'title': '[Full Ver.] M GIGS Ep. 59 - IDIOTAPE Live Part 1',
+            'creator': 'M-GIGS',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        stream_info = self._download_json(
+            'http://mwave.interest.me/onair/vod_info.m?vodtype=CL&sectorid=&endinfo=Y&id=%s' % video_id,
+            'Download stream info')
+
+        formats = []
+        for info in stream_info['cdn']:
+            f4m_stream = self._download_json(info['url'], video_id, 'Download f4m stream')
+            formats.extend(
+                self._extract_f4m_formats(f4m_stream['fileurl'] + '&g=PCROWKHLYUDY&hdcore=3.0.3', video_id))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': stream_info['title'],
+            'creator': stream_info.get('program_title'),
+            'formats': formats,
+        }

From 22c83245c51faa53118a8f815b13b2e4c2df9923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Aug 2015 23:07:41 +0600
Subject: [PATCH 1099/2145] [mwave] Improve

---
 youtube_dl/extractor/mwave.py | 50 ++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
index 7f91aa269..66b523197 100644
--- a/youtube_dl/extractor/mwave.py
+++ b/youtube_dl/extractor/mwave.py
@@ -1,46 +1,58 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
 class MwaveIE(InfoExtractor):
-    IE_NAME = 'mwave'
     _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
+        'md5': 'c930e27b7720aaa3c9d0018dfc8ff6cc',
         'info_dict': {
             'id': '168859',
             'ext': 'flv',
             'title': '[M COUNTDOWN] SISTAR - SHAKE IT',
-            'creator': 'M COUNTDOWN',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'M COUNTDOWN',
+            'duration': 206,
+            'view_count': int,
         }
-    }, {
-        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168860',
-        'info_dict': {
-            'id': '168860',
-            'ext': 'flv',
-            'title': '[Full Ver.] M GIGS Ep. 59 - IDIOTAPE Live Part 1',
-            'creator': 'M-GIGS',
-        }
-    }]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        stream_info = self._download_json(
+        vod_info = self._download_json(
             'http://mwave.interest.me/onair/vod_info.m?vodtype=CL&sectorid=&endinfo=Y&id=%s' % video_id,
-            'Download stream info')
+            video_id, 'Download vod JSON')
 
         formats = []
-        for info in stream_info['cdn']:
-            f4m_stream = self._download_json(info['url'], video_id, 'Download f4m stream')
+        for num, cdn_info in enumerate(vod_info['cdn']):
+            stream_url = cdn_info.get('url')
+            if not stream_url:
+                continue
+            stream_name = cdn_info.get('name') or compat_str(num)
+            f4m_stream = self._download_json(
+                stream_url, video_id,
+                'Download %s stream JSON' % stream_name)
+            f4m_url = f4m_stream.get('fileurl')
+            if not f4m_url:
+                continue
             formats.extend(
-                self._extract_f4m_formats(f4m_stream['fileurl'] + '&g=PCROWKHLYUDY&hdcore=3.0.3', video_id))
+                self._extract_f4m_formats(f4m_url + '&hdcore=3.0.3', video_id, f4m_id=stream_name))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': stream_info['title'],
-            'creator': stream_info.get('program_title'),
+            'title': vod_info['title'],
+            'thumbnail': vod_info.get('cover'),
+            'uploader': vod_info.get('program_title'),
+            'duration': parse_duration(vod_info.get('time')),
+            'view_count': int_or_none(vod_info.get('hit')),
             'formats': formats,
         }

From 26e1c3514f4af1ed60cd1114a653fe49e1fa8d11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:24:32 +0800
Subject: [PATCH 1100/2145] [theplatform] Add ThePlatformFeedIE

---
 youtube_dl/extractor/__init__.py    |   5 +-
 youtube_dl/extractor/theplatform.py | 162 +++++++++++++++++++++-------
 2 files changed, 126 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fa9acc923..b5f7ff9a9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -612,7 +612,10 @@ from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theonion import TheOnionIE
-from .theplatform import ThePlatformIE
+from .theplatform import (
+    ThePlatformIE,
+    ThePlatformFeedIE,
+)
 from .thesixtyone import TheSixtyOneIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 29f938a76..f02e0f58d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -15,13 +15,68 @@ from ..utils import (
     xpath_with_ns,
     unsmuggle_url,
     int_or_none,
+    url_basename,
+    float_or_none,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
-class ThePlatformIE(InfoExtractor):
+class ThePlatformBaseIE(InfoExtractor):
+    def _extract_theplatform_smil_formats(self, smil_url, video_id, note='Downloading SMIL data'):
+        meta = self._download_xml(smil_url, video_id, note=note)
+        try:
+            error_msg = next(
+                n.attrib['abstract']
+                for n in meta.findall(_x('.//smil:ref'))
+                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
+        except StopIteration:
+            pass
+        else:
+            raise ExtractorError(error_msg, expected=True)
+
+        formats = self._parse_smil_formats(
+            meta, smil_url, video_id, namespace=default_ns,
+            # the parameters are from syfy.com, other sites may use others,
+            # they also work for nbc.com
+            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
+            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
+
+        for _format in formats:
+            ext = determine_ext(_format['url'])
+            if ext == 'once':
+                _format['ext'] = 'mp4'
+
+        self._sort_formats(formats)
+
+        return formats
+
+    def get_metadata(self, path, video_id):
+        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
+        info_json = self._download_webpage(info_url, video_id)
+        info = json.loads(info_json)
+
+        subtitles = {}
+        captions = info.get('captions')
+        if isinstance(captions, list):
+            for caption in captions:
+                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
+                subtitles[lang] = [{
+                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
+                    'url': src,
+                }]
+
+        return {
+            'title': info['title'],
+            'subtitles': subtitles,
+            'description': info['description'],
+            'thumbnail': info['defaultThumbnailUrl'],
+            'duration': int_or_none(info.get('duration'), 1000),
+        }
+
+
+class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
@@ -118,51 +173,78 @@ class ThePlatformIE(InfoExtractor):
         if sig:
             smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
-        meta = self._download_xml(smil_url, video_id)
-        try:
-            error_msg = next(
-                n.attrib['abstract']
-                for n in meta.findall(_x('.//smil:ref'))
-                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
-        except StopIteration:
-            pass
-        else:
-            raise ExtractorError(error_msg, expected=True)
+        formats = self._extract_theplatform_smil_formats(smil_url, video_id)
 
-        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
-        info_json = self._download_webpage(info_url, video_id)
-        info = json.loads(info_json)
+        ret = self.get_metadata(path, video_id)
+        ret.update({
+            'id': video_id,
+            'formats': formats,
+        })
 
-        subtitles = {}
-        captions = info.get('captions')
-        if isinstance(captions, list):
-            for caption in captions:
-                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
-                subtitles[lang] = [{
-                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
-                    'url': src,
-                }]
+        return ret
 
-        formats = self._parse_smil_formats(
-            meta, smil_url, video_id, namespace=default_ns,
-            # the parameters are from syfy.com, other sites may use others,
-            # they also work for nbc.com
-            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
-            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
 
-        for _format in formats:
-            ext = determine_ext(_format['url'])
-            if ext == 'once':
-                _format['ext'] = 'mp4'
+class ThePlatformFeedIE(ThePlatformBaseIE):
+    _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&byGuid=%s'
+    _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*byGuid=(?P<id>[a-zA-Z0-9_]+)'
+    _TEST = {
+        # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
+        'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
+        'md5': '22d2b84f058d3586efcd99e57d59d314',
+        'info_dict': {
+            'id': 'n_hardball_5biden_140207',
+            'ext': 'mp4',
+            'title': 'The Biden factor: will Joe run in 2016?',
+            'description': 'Could Vice President Joe Biden be preparing a 2016 campaign? Mark Halperin and Sam Stein weigh in.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20140208',
+            'timestamp': 1391824260,
+            'duration': 467.0,
+            'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        provider_id = mobj.group('provider_id')
+        feed_id = mobj.group('feed_id')
+
+        real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, video_id)
+        feed = self._download_json(real_url, video_id)
+        entry = feed['entries'][0]
+
+        formats = []
+        first_video_id = None
+        duration = None
+        for item in entry['media$content']:
+            smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
+            cur_video_id = url_basename(smil_url)
+            if first_video_id is None:
+                first_video_id = cur_video_id
+                duration = float_or_none(item.get('plfile$duration'))
+            formats.extend(self._extract_theplatform_smil_formats(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id))
 
         self._sort_formats(formats)
 
-        return {
+        thumbnails = [{
+            'url': thumbnail['plfile$url'],
+            'width': int_or_none(thumbnail.get('plfile$width')),
+            'height': int_or_none(thumbnail.get('plfile$height')),
+        } for thumbnail in entry.get('media$thumbnails', [])]
+
+        timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
+        categories = [item['media$name'] for item in entry.get('media$categories', [])]
+
+        ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id)
+        ret.update({
             'id': video_id,
-            'title': info['title'],
-            'subtitles': subtitles,
             'formats': formats,
-            'description': info['description'],
-            'thumbnail': info['defaultThumbnailUrl'],
-            'duration': int_or_none(info.get('duration'), 1000),
-        }
+            'thumbnails': thumbnails,
+            'duration': duration,
+            'timestamp': timestamp,
+            'categories': categories,
+        })
+
+        return ret

From 05fe2594e4589b4e714a423550172eeec3949a70 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:38:39 +0800
Subject: [PATCH 1101/2145] [theplatform] Support URLs with 'guid='

---
 youtube_dl/extractor/theplatform.py | 36 +++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index f02e0f58d..883bf491c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -9,6 +9,10 @@ import hashlib
 
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -120,6 +124,20 @@ class ThePlatformIE(ThePlatformBaseIE):
     }, {
         'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
         'only_matching': True,
+    }, {
+        'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
+        'md5': '734f3790fb5fc4903da391beeebc4836',
+        'info_dict': {
+            'id': 'tdy_or_siri_150701',
+            'ext': 'mp4',
+            'title': 'iPhone Siri’s sassy response to a math question has people talking',
+            'description': 'md5:a565d1deadd5086f3331d57298ec6333',
+            'duration': 83.0,
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1435752600,
+            'upload_date': '20150701',
+            'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
+        },
     }]
 
     @staticmethod
@@ -154,6 +172,24 @@ class ThePlatformIE(ThePlatformBaseIE):
             path += '/media'
         path += '/' + video_id
 
+        qs_dict = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        if 'guid' in qs_dict:
+            webpage = self._download_webpage(url, video_id)
+            scripts = re.findall(r'<script[^>]+src="([^"]+)"', webpage)
+            feed_id = None
+            # feed id usually locates in the last script.
+            # Seems there's no pattern for the interested script filename, so
+            # I try one by one
+            for script in reversed(scripts):
+                feed_script = self._download_webpage(script, video_id, 'Downloading feed script')
+                feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
+                if feed_id is not None:
+                    break
+            if feed_id is None:
+                raise ExtractorError('Unable to find feed id')
+            return self.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % (
+                provider_id, feed_id, qs_dict['guid'][0]))
+
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
         elif mobj.group('config'):

From dac14bf311fd1b3c6af6c57b3b03878a11ef5aae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:41:18 +0800
Subject: [PATCH 1102/2145] [nbc] Add MSNBCIE

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b5f7ff9a9..86ea0576a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -358,6 +358,7 @@ from .nbc import (
     NBCNewsIE,
     NBCSportsIE,
     NBCSportsVPlayerIE,
+    MSNBCIE,
 )
 from .ndr import (
     NDRIE,
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index ccdbfb6c9..e683d24c4 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -236,3 +236,28 @@ class NBCNewsIE(InfoExtractor):
                 'url': info['videoAssets'][-1]['publicUrl'],
                 'ie_key': 'ThePlatform',
             }
+
+
+class MSNBCIE(InfoExtractor):
+    # https URLs redirect to corresponding http ones
+    _VALID_URL = r'http://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
+        'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
+        'info_dict': {
+            'id': 'n_hayes_Aimm_140801_272214',
+            'ext': 'mp4',
+            'title': 'The chaotic GOP immigration vote',
+            'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1406937606,
+            'upload_date': '20140802',
+            'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        embed_url = self._html_search_meta('embedURL', webpage)
+        return self.url_result(embed_url)

From aa6cd05ed82b14af0e3827b2ff43eed02087b574 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:47:55 +0800
Subject: [PATCH 1103/2145] [theplatform] Fix Python 2: declare coding

---
 youtube_dl/extractor/theplatform.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 883bf491c..adaec3375 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import re

From ce00af87670d47f4dff6ad80e46a29e49cbdfe4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Aug 2015 00:56:17 +0600
Subject: [PATCH 1104/2145] [extractor/common] Add default subtitles lang

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ac12be933..b1af45870 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1135,7 +1135,7 @@ class InfoExtractor(object):
 
         return formats
 
-    def _parse_smil_subtitles(self, smil, namespace=None):
+    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
         subtitles = {}
         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
             src = textstream.get('src')
@@ -1146,7 +1146,7 @@ class InfoExtractor(object):
                 type_ = textstream.get('type')
                 if type_ == 'text/srt':
                     ext = 'srt'
-            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName')
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,
                 'ext': ext,

From 5cdefc46257802708816e1d4ea7ff5cafe910ff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Aug 2015 01:02:50 +0600
Subject: [PATCH 1105/2145] [extractor/common] Add more subtitle mime types for
 guess when ext is missing

---
 youtube_dl/extractor/common.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b1af45870..ce2030d28 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1144,8 +1144,13 @@ class InfoExtractor(object):
             ext = textstream.get('ext') or determine_ext(src)
             if not ext:
                 type_ = textstream.get('type')
-                if type_ == 'text/srt':
-                    ext = 'srt'
+                SUBTITLES_TYPES = {
+                    'text/vtt': 'vtt',
+                    'text/srt': 'srt',
+                    'application/smptett+xml': 'tt',
+                }
+                if type_ in SUBTITLES_TYPES:
+                    ext = SUBTITLES_TYPES[type_]
             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,

From dd565ac1ad22fe48f8e358d95ea912b1768b1e5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 03:07:04 +0800
Subject: [PATCH 1106/2145] [theplatform] Use _download_json

---
 youtube_dl/extractor/theplatform.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index adaec3375..ba05ada39 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import time
 import hmac
 import binascii
@@ -59,8 +58,7 @@ class ThePlatformBaseIE(InfoExtractor):
 
     def get_metadata(self, path, video_id):
         info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
-        info_json = self._download_webpage(info_url, video_id)
-        info = json.loads(info_json)
+        info = self._download_json(info_url, video_id)
 
         subtitles = {}
         captions = info.get('captions')

From 061f62da54cb4184a039108e40dee8e9eb2611c1 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 20 Aug 2015 12:56:11 +0800
Subject: [PATCH 1107/2145] [vlive] New extractor for vlive.tv

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vlive.py    | 94 ++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 youtube_dl/extractor/vlive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1c53a5632..6bee5b63c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -735,6 +735,7 @@ from .vk import (
     VKIE,
     VKUserVideosIE,
 )
+from .vlive import VLiveIE
 from .vodlocker import VodlockerIE
 from .voicerepublic import VoiceRepublicIE
 from .vporn import VpornIE
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
new file mode 100644
index 000000000..b3bbd80fb
--- /dev/null
+++ b/youtube_dl/extractor/vlive.py
@@ -0,0 +1,94 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import hmac
+from hashlib import sha1
+from base64 import b64encode
+from time import time
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext
+)
+from ..compat import compat_urllib_parse
+
+
+class VLiveIE(InfoExtractor):
+    IE_NAME = 'vlive'
+    _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://m.vlive.tv/video/1326',
+        'md5': 'cc7314812855ce56de70a06a27314983',
+        'info_dict': {
+            'id': '1326',
+            'ext': 'mp4',
+            'title': '[V] Girl\'s Day\'s Broadcast',
+            'creator': 'Girl\'s Day',
+            'upload_date': '20150817',
+        },
+    }
+    _SECRET = 'rFkwZet6pqk1vQt6SxxUkAHX7YL3lmqzUMrU4IDusTo4jEBdtOhNfT4BYYAdArwH'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://m.vlive.tv/video/%s' % video_id,
+            video_id, note='Download video page')
+
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        creator = self._html_search_regex(
+            r'<span class="name">([^<>]+)</span>', webpage, 'creator')
+        upload_date = self._html_search_regex(
+            r'<span class="time">(\d{4}\.\d{2}\.\d{2})</span>', webpage,
+            'upload date', default=None, fatal=False)
+        if upload_date:
+            upload_date = upload_date.replace('.', '')
+
+        url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
+        msgpad = {'msgpad': '%.0f' % (time() * 1000)}
+        md = {
+            'md': b64encode(
+                hmac.new(self._SECRET.encode('ascii'),
+                         (url[:255] + msgpad['msgpad']).encode('ascii'), sha1).digest())
+        }
+        url += '&' + compat_urllib_parse.urlencode(msgpad) + '&' + compat_urllib_parse.urlencode(md)
+
+        playinfo = self._download_json(url, video_id, 'Downloading video json')
+
+        if playinfo.get('message', '') != 'success':
+            raise ExtractorError(playinfo['message'])
+
+        if not playinfo.get('result'):
+            raise ExtractorError('No videos found.')
+
+        formats = []
+        for vid in playinfo['result'].get('videos', {}).get('list', []):
+            formats.append({
+                'url': vid['source'],
+                'ext': 'mp4',
+                'abr': vid.get('bitrate', {}).get('audio'),
+                'vbr': vid.get('bitrate', {}).get('video'),
+                'format_id': vid['encodingOption']['name'],
+                'height': vid.get('height'),
+                'width': vid.get('width'),
+            })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for caption in playinfo['result'].get('captions', {}).get('list', []):
+            subtitles[caption['language']] = [
+                {'ext': determine_ext(caption['source'], default_ext='vtt'),
+                 'url': caption['source']}]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'creator': creator,
+            'thumbnail': thumbnail,
+            'formats': formats,
+            'upload_date': upload_date,
+            'subtitles': subtitles,
+        }

From 03bc7237add1747de4c0c5d09e72e03639b4fd21 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 23:18:58 +0800
Subject: [PATCH 1108/2145] [common] _parse_smil_subtitles: accept `lang` as
 the subtitle language

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ce2030d28..999afc110 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1151,7 +1151,7 @@ class InfoExtractor(object):
                 }
                 if type_ in SUBTITLES_TYPES:
                     ext = SUBTITLES_TYPES[type_]
-            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or subtitles_lang
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,
                 'ext': ext,

From 912e0b7e46d795df3ec1866f9b0ff071cca8d550 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:37:07 +0800
Subject: [PATCH 1109/2145] [common] Add _merge_subtitles()

---
 youtube_dl/extractor/common.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 999afc110..b7437af5a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1279,6 +1279,26 @@ class InfoExtractor(object):
     def _get_subtitles(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
+    @staticmethod
+    def _merge_subtitle_items(subtitle_list1, subtitle_list2):
+        """ Merge subtitle items for one language. Items with duplicated URLs
+        will be dropped. """
+        list1_urls = set([item['url'] for item in subtitle_list1])
+        ret = list(subtitle_list1)
+        ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
+        return ret
+
+    @classmethod
+    def _merge_subtitles(kls, subtitle_dict1, subtitle_dict2):
+        """ Merge two subtitle dictionaries, language by language. """
+        print(subtitle_dict1)
+        print(subtitle_dict2)
+        ret = dict(subtitle_dict1)
+        for lang in subtitle_dict2:
+            ret[lang] = kls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
+        print(ret)
+        return ret
+
     def extract_automatic_captions(self, *args, **kwargs):
         if (self._downloader.params.get('writeautomaticsub', False) or
                 self._downloader.params.get('listsubtitles')):

From c687ac745b3c94b2fd246214e78c92a31bd9fc0f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:37:43 +0800
Subject: [PATCH 1110/2145] [theplatform] Use subtitles from SMIL, too

---
 youtube_dl/extractor/theplatform.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index ba05ada39..25edc3100 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -28,7 +28,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
 class ThePlatformBaseIE(InfoExtractor):
-    def _extract_theplatform_smil_formats(self, smil_url, video_id, note='Downloading SMIL data'):
+    def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note)
         try:
             error_msg = next(
@@ -54,7 +54,9 @@ class ThePlatformBaseIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        return formats
+        subtitles = self._parse_smil_subtitles(meta, default_ns)
+
+        return formats, subtitles
 
     def get_metadata(self, path, video_id):
         info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
@@ -208,12 +210,14 @@ class ThePlatformIE(ThePlatformBaseIE):
         if sig:
             smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
-        formats = self._extract_theplatform_smil_formats(smil_url, video_id)
+        formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
 
         ret = self.get_metadata(path, video_id)
+        combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
         ret.update({
             'id': video_id,
             'formats': formats,
+            'subtitles': combined_subtitles,
         })
 
         return ret
@@ -251,6 +255,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         entry = feed['entries'][0]
 
         formats = []
+        subtitles = {}
         first_video_id = None
         duration = None
         for item in entry['media$content']:
@@ -259,7 +264,9 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             if first_video_id is None:
                 first_video_id = cur_video_id
                 duration = float_or_none(item.get('plfile$duration'))
-            formats.extend(self._extract_theplatform_smil_formats(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id))
+            cur_formats, cur_subtitles = self._extract_theplatform_smil(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id)
+            formats.extend(cur_formats)
+            subtitles = self._merge_subtitles(subtitles, cur_subtitles)
 
         self._sort_formats(formats)
 
@@ -273,9 +280,11 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         categories = [item['media$name'] for item in entry.get('media$categories', [])]
 
         ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id)
+        subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
         ret.update({
             'id': video_id,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnails': thumbnails,
             'duration': duration,
             'timestamp': timestamp,

From f908b74fa38b2678e26aea128dbd934cd781a9b6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:38:57 +0800
Subject: [PATCH 1111/2145] [test/subtitles] Add test for ThePlatformFeedIE

---
 test/test_subtitles.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c4e3adb67..0343967d9 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -25,6 +25,7 @@ from youtube_dl.extractor import (
     RaiIE,
     VikiIE,
     ThePlatformIE,
+    ThePlatformFeedIE,
     RTVEALaCartaIE,
     FunnyOrDieIE,
 )
@@ -307,6 +308,18 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
 
 
+class TestThePlatformFeedSubtitles(BaseTestSubtitles):
+    url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
+    IE = ThePlatformFeedIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), '48649a22e82b2da21c9a67a395eedade')
+
+
 class TestRtveSubtitles(BaseTestSubtitles):
     url = 'http://www.rtve.es/alacarta/videos/los-misterios-de-laura/misterios-laura-capitulo-32-misterio-del-numero-17-2-parte/2428621/'
     IE = RTVEALaCartaIE

From f738dd7b7c7aefe4d26a65905dee9567a691d262 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:43:22 +0800
Subject: [PATCH 1112/2145] [common] Remove debugging codes

---
 youtube_dl/extractor/common.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b7437af5a..f731703fb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1291,12 +1291,9 @@ class InfoExtractor(object):
     @classmethod
     def _merge_subtitles(kls, subtitle_dict1, subtitle_dict2):
         """ Merge two subtitle dictionaries, language by language. """
-        print(subtitle_dict1)
-        print(subtitle_dict2)
         ret = dict(subtitle_dict1)
         for lang in subtitle_dict2:
             ret[lang] = kls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
-        print(ret)
         return ret
 
     def extract_automatic_captions(self, *args, **kwargs):

From dc95bd503e82d3eb04c347ac0cdbcbabd7e14552 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Aug 2015 08:54:28 +0600
Subject: [PATCH 1113/2145] [folketinget] Add skip_download for test

---
 youtube_dl/extractor/folketinget.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dl/extractor/folketinget.py
index 0fb29de75..75399fa7d 100644
--- a/youtube_dl/extractor/folketinget.py
+++ b/youtube_dl/extractor/folketinget.py
@@ -30,6 +30,10 @@ class FolketingetIE(InfoExtractor):
             'upload_date': '20141120',
             'duration': 3960,
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 5d003e29b188dd2f140fe1b9b93f1bb1ad8263a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Aug 2015 08:56:05 +0600
Subject: [PATCH 1114/2145] [rtp] Add skip_download for test

---
 youtube_dl/extractor/rtp.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index ecf4939cd..82b323cdd 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -18,6 +18,10 @@ class RTPIE(InfoExtractor):
             'description': 'As paixões musicais de António Cartaxo e António Macedo',
             'thumbnail': 're:^https?://.*\.jpg',
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
         'only_matching': True,

From 4932a817a0c2375df14d66c9ac86cfa28988327d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:00:08 +0800
Subject: [PATCH 1115/2145] [rtl2] Add skip_download for test

---
 youtube_dl/extractor/rtl2.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 72cd80498..e9589449e 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -15,6 +15,10 @@ class RTL2IE(InfoExtractor):
             'title': 'GRIP sucht den Sommerkönig',
             'description': 'Matthias, Det und Helge treten gegeneinander an.'
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
         'md5': 'ffcd517d2805b57ce11a58a2980c2b02',

From 9eb4ab6ad915a777b6f7d7b39d03d05d7d31cd24 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:04:25 +0800
Subject: [PATCH 1116/2145] [rtl2] Remove an unused line

---
 youtube_dl/extractor/rtl2.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index e9589449e..9e0c6890e 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -43,7 +43,6 @@ class RTL2IE(InfoExtractor):
         vivi_id = self._html_search_regex(
             r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
-        webpage = self._download_webpage(info_url, '')
 
         info = self._download_json(info_url, video_id)
         video_info = info['video']

From 5e1a5ac8de12391cb22d2fa0dfb2119527bd7fc2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:20:32 +0800
Subject: [PATCH 1117/2145] [rtl2] Fix extraction for test_RTL2_1

---
 youtube_dl/extractor/rtl2.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 9e0c6890e..276612fc7 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,6 +1,7 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
 from .common import InfoExtractor
 
 
@@ -28,6 +29,10 @@ class RTL2IE(InfoExtractor):
             'title': 'Anna erwischt Alex!',
             'description': 'Anna ist Alex\' Tochter bei Köln 50667.'
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -38,10 +43,17 @@ class RTL2IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        vico_id = self._html_search_regex(
-            r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
-        vivi_id = self._html_search_regex(
-            r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
+        mobj = re.search(
+            r'<div[^>]+data-collection="(?P<vico_id>\d+)"[^>]+data-video="(?P<vivi_id>\d+)"',
+            webpage)
+        if mobj:
+            vico_id = mobj.group('vico_id')
+            vivi_id = mobj.group('vivi_id')
+        else:
+            vico_id = self._html_search_regex(
+                r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
+            vivi_id = self._html_search_regex(
+                r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
 
         info = self._download_json(info_url, video_id)

From d7c16305707f2af5c47d91b67cf0850b4dcada3a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:21:21 +0800
Subject: [PATCH 1118/2145] [rtl2] Remove MD5 checksums

---
 youtube_dl/extractor/rtl2.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 276612fc7..25f7faf76 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -9,7 +9,6 @@ class RTL2IE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
         'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
-        'md5': 'bfcc179030535b08dc2b36b469b5adc7',
         'info_dict': {
             'id': 'folge-203-0',
             'ext': 'f4v',
@@ -22,7 +21,6 @@ class RTL2IE(InfoExtractor):
         },
     }, {
         'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
-        'md5': 'ffcd517d2805b57ce11a58a2980c2b02',
         'info_dict': {
             'id': '21040-anna-erwischt-alex',
             'ext': 'mp4',

From 8c97f81943de1c2bf8d2f524ba5ca09b29579dc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 21 Aug 2015 11:35:51 +0200
Subject: [PATCH 1119/2145] [common] Follow convention of using 'cls' in
 classmethods

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f731703fb..5d24bcb6a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1289,11 +1289,11 @@ class InfoExtractor(object):
         return ret
 
     @classmethod
-    def _merge_subtitles(kls, subtitle_dict1, subtitle_dict2):
+    def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
         """ Merge two subtitle dictionaries, language by language. """
         ret = dict(subtitle_dict1)
         for lang in subtitle_dict2:
-            ret[lang] = kls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
+            ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
         return ret
 
     def extract_automatic_captions(self, *args, **kwargs):

From 66ce97024d0de7836777562a6eb60603796636d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Aug 2015 06:30:00 +0600
Subject: [PATCH 1120/2145] [soundcloud:user] Update tests

---
 youtube_dl/extractor/soundcloud.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 6ce86cbcd..ed5dcc0d3 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -309,7 +309,7 @@ class SoundcloudUserIE(SoundcloudIE):
             'id': '114582580',
             'title': 'The Akashic Chronicler (All)',
         },
-        'playlist_mincount': 112,
+        'playlist_mincount': 111,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
         'info_dict': {
@@ -330,14 +330,14 @@ class SoundcloudUserIE(SoundcloudIE):
             'id': '114582580',
             'title': 'The Akashic Chronicler (Reposts)',
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 7,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
         'info_dict': {
             'id': '114582580',
             'title': 'The Akashic Chronicler (Likes)',
         },
-        'playlist_mincount': 333,
+        'playlist_mincount': 321,
     }, {
         'url': 'https://soundcloud.com/grynpyret/spotlight',
         'info_dict': {

From 483fc223bb1509d11ac1843a5852f75c0aec3475 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 10:42:34 +0600
Subject: [PATCH 1121/2145] [pluralsight] Add extractor (Closes #6090)

---
 youtube_dl/extractor/__init__.py    |   4 +
 youtube_dl/extractor/pluralsight.py | 218 ++++++++++++++++++++++++++++
 2 files changed, 222 insertions(+)
 create mode 100644 youtube_dl/extractor/pluralsight.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1c53a5632..d59882598 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -454,6 +454,10 @@ from .playfm import PlayFMIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
+from .pluralsight import (
+    PluralsightIE,
+    PluralsightCourseIE,
+)
 from .podomatic import PodomaticIE
 from .porn91 import Porn91IE
 from .pornhd import PornHdIE
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
new file mode 100644
index 000000000..1bdcacbaa
--- /dev/null
+++ b/youtube_dl/extractor/pluralsight.py
@@ -0,0 +1,218 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_duration,
+)
+
+
+class PluralsightIE(InfoExtractor):
+    IE_NAME = 'pluralsight'
+    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
+    _LOGIN_URL = 'https://www.pluralsight.com/id/'
+    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
+    _NETRC_MACHINE = 'pluralsight'
+
+    _TEST = {
+        'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
+        'md5': '4d458cf5cf4c593788672419a8dd4cf8',
+        'info_dict': {
+            'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
+            'ext': 'mp4',
+            'title': 'Management of SQL Server - Demo Monitoring',
+            'duration': 338,
+        },
+        'skip': 'Requires pluralsight account credentials',
+    }
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            raise ExtractorError(
+                'Pluralsight account is required, use --username and --password options to provide account credentials.',
+                expected=True)
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'Username': username.encode('utf-8'),
+            'Password': password.encode('utf-8'),
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+        request = compat_urllib_request.Request(
+            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        error = self._search_regex(
+            r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
+            response, 'error message', default=None)
+        if error:
+            raise ExtractorError('Unable to login: %s' % error, expected=True)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        author = mobj.group('author')
+        name = mobj.group('name')
+        clip_id = mobj.group('clip')
+        course = mobj.group('course')
+
+        display_id = '%s-%s' % (name, clip_id)
+
+        webpage = self._download_webpage(url, display_id)
+
+        collection = self._parse_json(
+            self._search_regex(
+                r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
+                webpage, 'modules'),
+            display_id)
+
+        module, clip = None, None
+
+        for module_ in collection:
+            if module_.get('moduleName') == name:
+                module = module_
+                for clip_ in module_.get('clips', []):
+                    clip_index = clip_.get('clipIndex')
+                    if clip_index is None:
+                        continue
+                    if compat_str(clip_index) == clip_id:
+                        clip = clip_
+                        break
+
+        if not clip:
+            raise ExtractorError('Unable to resolve clip')
+
+        QUALITIES = {
+            'low': {'width': 640, 'height': 480},
+            'medium': {'width': 848, 'height': 640},
+            'high': {'width': 1024, 'height': 768},
+        }
+
+        ALLOWED_QUALITIES = (
+            ('webm', ('high',)),
+            ('mp4', ('low', 'medium', 'high',)),
+        )
+
+        formats = []
+        for ext, qualities in ALLOWED_QUALITIES:
+            for quality in qualities:
+                f = QUALITIES[quality].copy()
+                clip_post = {
+                    'a': author,
+                    'cap': 'false',
+                    'cn': clip_id,
+                    'course': course,
+                    'lc': 'en',
+                    'm': name,
+                    'mt': ext,
+                    'q': '%dx%d' % (f['width'], f['height']),
+                }
+                request = compat_urllib_request.Request(
+                    'http://www.pluralsight.com/training/Player/ViewClip',
+                    json.dumps(clip_post).encode('utf-8'))
+                request.add_header('Content-Type', 'application/json;charset=utf-8')
+                format_id = '%s-%s' % (ext, quality)
+                clip_url = self._download_webpage(
+                    request, display_id, 'Downloading %s URL' % format_id, fatal=False)
+                if not clip_url:
+                    continue
+                f.update({
+                    'url': clip_url,
+                    'ext': ext,
+                    'format_id': format_id,
+                })
+                formats.append(f)
+        self._sort_formats(formats)
+
+        # TODO: captions
+        # http://www.pluralsight.com/training/Player/ViewClip + cap = true
+        # or
+        # http://www.pluralsight.com/training/Player/Captions
+        # { a = author, cn = clip_id, lc = end, m = name }
+
+        return {
+            'id': clip['clipName'],
+            'title': '%s - %s' % (module['title'], clip['title']),
+            'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
+            'creator': author,
+            'formats': formats
+        }
+
+
+class PluralsightCourseIE(InfoExtractor):
+    IE_NAME = 'pluralsight:course'
+    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
+    _TEST = {
+        # Free course from Pluralsight Starter Subscription for Microsoft TechNet
+        # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
+        'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
+        'info_dict': {
+            'id': 'hosting-sql-server-windows-azure-iaas',
+            'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
+            'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
+        },
+        'playlist_count': 31,
+    }
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+
+        course = self._download_json(
+            'http://www.pluralsight.com/data/course/%s' % course_id,
+            course_id, 'Downloading course JSON')
+
+        title = course['title']
+        description = course.get('description') or course.get('shortDescription')
+
+        course_data = self._download_json(
+            'http://www.pluralsight.com/data/course/content/%s' % course_id,
+            course_id, 'Downloading course data JSON')
+
+        may_not_view = 0
+
+        entries = []
+        for module in course_data:
+            for clip in module.get('clips', []):
+                if clip.get('userMayViewClip') is False:
+                    may_not_view += 1
+                    continue
+                player_parameters = clip.get('playerParameters')
+                if not player_parameters:
+                    continue
+                entries.append(self.url_result(
+                    'http://www.pluralsight.com/training/player?%s' % player_parameters,
+                    'Pluralsight'))
+
+        if may_not_view > 0:
+            self._downloader.report_warning(
+                'There are %d videos in this course that are not available for you. '
+                'Upgrade your account to get access to these videos.' % may_not_view)
+
+        return self.playlist_result(entries, course_id, title, description)

From 468083d2f5596314a0813859f3afe7d2fce3cac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 10:44:10 +0600
Subject: [PATCH 1122/2145] [pluralsight] Remove unused const

---
 youtube_dl/extractor/pluralsight.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 1bdcacbaa..7c7f762ff 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -21,7 +21,6 @@ class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
     _LOGIN_URL = 'https://www.pluralsight.com/id/'
-    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'pluralsight'
 
     _TEST = {

From 2b6bda1ed86e1b64242b33c032286dc315d541ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 11:21:56 +0600
Subject: [PATCH 1123/2145] [pluralsight] Do not yet rely on userMayViewClip

---
 youtube_dl/extractor/pluralsight.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7c7f762ff..7ba396aef 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -183,6 +183,8 @@ class PluralsightCourseIE(InfoExtractor):
     def _real_extract(self, url):
         course_id = self._match_id(url)
 
+        # TODO: PSM cookie
+
         course = self._download_json(
             'http://www.pluralsight.com/data/course/%s' % course_id,
             course_id, 'Downloading course JSON')
@@ -194,14 +196,9 @@ class PluralsightCourseIE(InfoExtractor):
             'http://www.pluralsight.com/data/course/content/%s' % course_id,
             course_id, 'Downloading course data JSON')
 
-        may_not_view = 0
-
         entries = []
         for module in course_data:
             for clip in module.get('clips', []):
-                if clip.get('userMayViewClip') is False:
-                    may_not_view += 1
-                    continue
                 player_parameters = clip.get('playerParameters')
                 if not player_parameters:
                     continue
@@ -209,9 +206,4 @@ class PluralsightCourseIE(InfoExtractor):
                     'http://www.pluralsight.com/training/player?%s' % player_parameters,
                     'Pluralsight'))
 
-        if may_not_view > 0:
-            self._downloader.report_warning(
-                'There are %d videos in this course that are not available for you. '
-                'Upgrade your account to get access to these videos.' % may_not_view)
-
         return self.playlist_result(entries, course_id, title, description)

From 2006a06eff606c5a996c315a3e597b9d2603db9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 21:43:28 +0600
Subject: [PATCH 1124/2145] [kontrtube] Fix extraction (Closes #6644)

---
 youtube_dl/extractor/kontrtube.py | 40 +++++++++++++++----------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index 720bc939b..a59c529f4 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
 class KontrTubeIE(InfoExtractor):
@@ -34,33 +37,28 @@ class KontrTubeIE(InfoExtractor):
         webpage = self._download_webpage(
             url, display_id, 'Downloading page')
 
-        video_url = self._html_search_regex(
+        video_url = self._search_regex(
             r"video_url\s*:\s*'(.+?)/?',", webpage, 'video URL')
-        thumbnail = self._html_search_regex(
-            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'video thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'thumbnail', fatal=False)
         title = self._html_search_regex(
-            r'<title>(.+?)</title>', webpage, 'video title')
+            r'(?s)<h2>(.+?)</h2>', webpage, 'title')
         description = self._html_search_meta(
-            'description', webpage, 'video description')
+            'description', webpage, 'description')
 
-        mobj = re.search(
-            r'<div class="col_2">Длительность: <span>(?P<minutes>\d+)м:(?P<seconds>\d+)с</span></div>',
-            webpage)
-        duration = int(mobj.group('minutes')) * 60 + int(mobj.group('seconds')) if mobj else None
+        duration = self._search_regex(
+            r'Длительность: <em>([^<]+)</em>', webpage, 'duration', fatal=False)
+        if duration:
+            duration = parse_duration(duration.replace('мин', 'min').replace('сек', 'sec'))
 
-        view_count = self._html_search_regex(
-            r'<div class="col_2">Просмотров: <span>(\d+)</span></div>',
+        view_count = self._search_regex(
+            r'Просмотров: <em>([^<]+)</em>',
             webpage, 'view count', fatal=False)
+        if view_count:
+            view_count = int_or_none(view_count.replace(' ', ''))
 
-        comment_count = None
-        comment_str = self._html_search_regex(
-            r'Комментарии: <span>([^<]+)</span>', webpage, 'comment count', fatal=False)
-        if comment_str.startswith('комментариев нет'):
-            comment_count = 0
-        else:
-            mobj = re.search(r'\d+ из (?P<total>\d+) комментариев', comment_str)
-            if mobj:
-                comment_count = mobj.group('total')
+        comment_count = int_or_none(self._search_regex(
+            r'Комментарии \((\d+)\)<', webpage, ' comment count', fatal=False))
 
         return {
             'id': video_id,

From 9990c960f2d944cfbecb7d613062b98fe99464a7 Mon Sep 17 00:00:00 2001
From: clauderains <clauderains3319@gmail.com>
Date: Sun, 23 Aug 2015 02:46:29 -0700
Subject: [PATCH 1125/2145] [spankwire] Fixed uploader_id field extraction so
 that test case passes

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 5fa6faf18..0a35c2b3b 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -54,7 +54,7 @@ class SpankwireIE(InfoExtractor):
             r'by:\s*<a [^>]*>(.+?)</a>',
             webpage, 'uploader', fatal=False)
         uploader_id = self._html_search_regex(
-            r'by:\s*<a href="/Profile\.aspx\?.*?UserId=(\d+).*?"',
+            r'by:\s*<a href="/user/viewProfile\?.*?UserId=(\d+).*?"',
             webpage, 'uploader id', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'</a> on (.+?) at \d+:\d+',

From 59e6acc757a9df85ca78e519623e84072ffd9c01 Mon Sep 17 00:00:00 2001
From: clauderains <clauderains3319@gmail.com>
Date: Sun, 23 Aug 2015 02:47:20 -0700
Subject: [PATCH 1126/2145] [spankwire] Support new cdn video url format

---
 youtube_dl/extractor/spankwire.py | 84 +++++++++++++++++++++----------
 1 file changed, 58 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 0a35c2b3b..0f2d8d0de 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -17,20 +17,34 @@ from ..aes import aes_decrypt_text
 
 class SpankwireIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
-    _TEST = {
-        'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
-        'md5': '8bbfde12b101204b39e4b9fe7eb67095',
-        'info_dict': {
-            'id': '103545',
-            'ext': 'mp4',
-            'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
-            'description': 'Crazy Bitch X rated music video.',
-            'uploader': 'oreusz',
-            'uploader_id': '124697',
-            'upload_date': '20070507',
-            'age_limit': 18,
-        }
-    }
+    _TESTS = [{
+                'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
+                'md5': '8bbfde12b101204b39e4b9fe7eb67095',
+                    'info_dict': {
+                                    'id': '103545',
+                                    'ext': 'mp4',
+                                    'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
+                                    'description': 'Crazy Bitch X rated music video.',
+                                    'uploader': 'oreusz',
+                                    'uploader_id': '124697',
+                                    'upload_date': '20070507',
+                                    'age_limit': 18,
+                    }
+              },
+              {
+                'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
+                'md5': '09b3c20833308b736ae8902db2f8d7e6',
+                    'info_dict': {
+                                    'id': '1921551',
+                                    'ext': 'mp4',
+                                    'title': 'Titcums Compiloation I',
+                                    'description': 'cum on tits',
+                                    'uploader': 'dannyh78999',
+                                    'uploader_id': '3056053',
+                                    'upload_date': '20150822',
+                                    'age_limit': 18,
+                    }
+              }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -82,18 +96,36 @@ class SpankwireIE(InfoExtractor):
         for video_url in video_urls:
             path = compat_urllib_parse_urlparse(video_url).path
             format = path.split('/')[4].split('_')[:2]
-            resolution, bitrate_str = format
-            format = "-".join(format)
-            height = int(resolution.rstrip('Pp'))
-            tbr = int(bitrate_str.rstrip('Kk'))
-            formats.append({
-                'url': video_url,
-                'resolution': resolution,
-                'format': format,
-                'tbr': tbr,
-                'height': height,
-                'format_id': format,
-            })
+            if format[0] == 'mp4':
+                format_id, quality = format
+                format = "-".join(format)
+                if quality == 'normal':
+                    height = 180
+                elif quality == 'high':
+                    height = 240
+                elif quality == 'ultra':
+                    height = 480
+                elif quality == '720p':
+                    height = 720
+                formats.append({
+                    'url': video_url,
+                    'format': format,
+                    'height': height,
+                    'format_id': format,
+                })
+            else:
+                resolution, bitrate_str = format
+                format = "-".join(format)
+                height = int(resolution.rstrip('Pp'))
+                tbr = int(bitrate_str.rstrip('Kk'))
+                formats.append({
+                    'url': video_url,
+                    'resolution': resolution,
+                    'format': format,
+                    'tbr': tbr,
+                    'height': height,
+                    'format_id': format,
+                })
         self._sort_formats(formats)
 
         age_limit = self._rta_search(webpage)

From 551c7837ace81190ce9141551dceec24dfdae1bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 22:32:20 +0600
Subject: [PATCH 1127/2145] [spankwire] Simplify and properly format

---
 youtube_dl/extractor/spankwire.py | 105 +++++++++++++-----------------
 1 file changed, 45 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 0f2d8d0de..0a47441b1 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -18,33 +18,34 @@ from ..aes import aes_decrypt_text
 class SpankwireIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
     _TESTS = [{
-                'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
-                'md5': '8bbfde12b101204b39e4b9fe7eb67095',
-                    'info_dict': {
-                                    'id': '103545',
-                                    'ext': 'mp4',
-                                    'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
-                                    'description': 'Crazy Bitch X rated music video.',
-                                    'uploader': 'oreusz',
-                                    'uploader_id': '124697',
-                                    'upload_date': '20070507',
-                                    'age_limit': 18,
-                    }
-              },
-              {
-                'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
-                'md5': '09b3c20833308b736ae8902db2f8d7e6',
-                    'info_dict': {
-                                    'id': '1921551',
-                                    'ext': 'mp4',
-                                    'title': 'Titcums Compiloation I',
-                                    'description': 'cum on tits',
-                                    'uploader': 'dannyh78999',
-                                    'uploader_id': '3056053',
-                                    'upload_date': '20150822',
-                                    'age_limit': 18,
-                    }
-              }]
+        # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
+        'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
+        'md5': '8bbfde12b101204b39e4b9fe7eb67095',
+        'info_dict': {
+            'id': '103545',
+            'ext': 'mp4',
+            'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
+            'description': 'Crazy Bitch X rated music video.',
+            'uploader': 'oreusz',
+            'uploader_id': '124697',
+            'upload_date': '20070507',
+            'age_limit': 18,
+        }
+    }, {
+        # download URL pattern: */mp4_<format_id>_<video_id>.mp4
+        'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
+        'md5': '09b3c20833308b736ae8902db2f8d7e6',
+        'info_dict': {
+            'id': '1921551',
+            'ext': 'mp4',
+            'title': 'Titcums Compiloation I',
+            'description': 'cum on tits',
+            'uploader': 'dannyh78999',
+            'uploader_id': '3056053',
+            'upload_date': '20150822',
+            'age_limit': 18,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -81,9 +82,10 @@ class SpankwireIE(InfoExtractor):
             r'<span\s+id="spCommentCount"[^>]*>([\d,\.]+)</span>',
             webpage, 'comment count', fatal=False))
 
-        video_urls = list(map(
-            compat_urllib_parse_unquote,
-            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
+        videos = re.findall(
+            r'playerData\.cdnPath([0-9]{3,})\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)
+        heights = [int(video[0]) for video in videos]
+        video_urls = list(map(compat_urllib_parse_unquote, [video[1] for video in videos]))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',
@@ -93,39 +95,22 @@ class SpankwireIE(InfoExtractor):
                 video_urls))
 
         formats = []
-        for video_url in video_urls:
+        for height, video_url in zip(heights, video_urls):
             path = compat_urllib_parse_urlparse(video_url).path
-            format = path.split('/')[4].split('_')[:2]
-            if format[0] == 'mp4':
-                format_id, quality = format
-                format = "-".join(format)
-                if quality == 'normal':
-                    height = 180
-                elif quality == 'high':
-                    height = 240
-                elif quality == 'ultra':
-                    height = 480
-                elif quality == '720p':
-                    height = 720
-                formats.append({
-                    'url': video_url,
-                    'format': format,
-                    'height': height,
-                    'format_id': format,
+            _, quality = path.split('/')[4].split('_')[:2]
+            f = {
+                'url': video_url,
+                'height': height,
+            }
+            tbr = self._search_regex(r'^(\d+)[Kk]$', quality, 'tbr', default=None)
+            if tbr:
+                f.update({
+                    'tbr': int(tbr),
+                    'format_id': '%dp' % height,
                 })
             else:
-                resolution, bitrate_str = format
-                format = "-".join(format)
-                height = int(resolution.rstrip('Pp'))
-                tbr = int(bitrate_str.rstrip('Kk'))
-                formats.append({
-                    'url': video_url,
-                    'resolution': resolution,
-                    'format': format,
-                    'tbr': tbr,
-                    'height': height,
-                    'format_id': format,
-                })
+                f['format_id'] = quality
+            formats.append(f)
         self._sort_formats(formats)
 
         age_limit = self._rta_search(webpage)

From 28b83495d898530e72d242874576f4d2d6d8ab3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 22:32:44 +0600
Subject: [PATCH 1128/2145] [spankwire] Simplify

---
 youtube_dl/extractor/spankwire.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 0a47441b1..609f78294 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -16,7 +16,7 @@ from ..aes import aes_decrypt_text
 
 
 class SpankwireIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<id>[0-9]+)/?)'
     _TESTS = [{
         # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
         'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
@@ -49,10 +49,9 @@ class SpankwireIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-        url = 'http://www.' + mobj.group('url')
+        video_id = mobj.group('id')
 
-        req = compat_urllib_request.Request(url)
+        req = compat_urllib_request.Request('http://www.' + mobj.group('url'))
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 90076b6172f25a36ca2a00c1b85cda169f2133c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 22:33:26 +0600
Subject: [PATCH 1129/2145] [spankwire] Preserve old uploader pattern

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 609f78294..9e8fb35b2 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -68,7 +68,7 @@ class SpankwireIE(InfoExtractor):
             r'by:\s*<a [^>]*>(.+?)</a>',
             webpage, 'uploader', fatal=False)
         uploader_id = self._html_search_regex(
-            r'by:\s*<a href="/user/viewProfile\?.*?UserId=(\d+).*?"',
+            r'by:\s*<a href="/(?:user/viewProfile|Profile\.aspx)\?.*?UserId=(\d+).*?"',
             webpage, 'uploader id', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'</a> on (.+?) at \d+:\d+',

From e7c14660d3aef3a5a303a82dee7e11dfe063048d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Aug 2015 00:36:24 +0600
Subject: [PATCH 1130/2145] [yandexmusic] Defer link resolve till actual
 download time to prevent link expiry (Closes #6650)

---
 youtube_dl/extractor/yandexmusic.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index f4c0f5702..85c495c11 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -1,4 +1,4 @@
-# coding=utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -64,7 +64,15 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
         return self._get_track_info(track)
 
 
-class YandexMusicAlbumIE(YandexMusicBaseIE):
+class YandexMusicPlaylistBaseIE(InfoExtractor):
+    def _build_playlist(self, tracks):
+        return [
+            self.url_result(
+                'http://music.yandex.ru/album/%s/track/%s' % (track['albums'][0]['id'], track['id']))
+            for track in tracks]
+
+
+class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):
     IE_NAME = 'yandexmusic:album'
     IE_DESC = 'Яндекс.Музыка - Альбом'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<id>\d+)/?(\?|$)'
@@ -85,7 +93,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE):
             'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id,
             album_id, 'Downloading album JSON')
 
-        entries = [self._get_track_info(track) for track in album['volumes'][0]]
+        entries = self._build_playlist(album['volumes'][0])
 
         title = '%s - %s' % (album['artists'][0]['name'], album['title'])
         year = album.get('year')
@@ -95,7 +103,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE):
         return self.playlist_result(entries, compat_str(album['id']), title)
 
 
-class YandexMusicPlaylistIE(YandexMusicBaseIE):
+class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
     IE_NAME = 'yandexmusic:playlist'
     IE_DESC = 'Яндекс.Музыка - Плейлист'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
@@ -120,8 +128,7 @@ class YandexMusicPlaylistIE(YandexMusicBaseIE):
                 r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
             playlist_id)['pageData']['playlist']
 
-        entries = [self._get_track_info(track) for track in playlist['tracks']]
-
         return self.playlist_result(
-            entries, compat_str(playlist_id),
+            self._build_playlist(playlist['tracks']),
+            compat_str(playlist_id),
             playlist['title'], playlist.get('description'))

From e4df2f98ccbe2e24785dd6883d7fd495193fd8e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Aug 2015 00:36:54 +0600
Subject: [PATCH 1131/2145] [yandexmusic:track] Eliminate base class

---
 youtube_dl/extractor/yandexmusic.py | 36 ++++++++++++++---------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 85c495c11..91829be1c 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -12,7 +12,23 @@ from ..utils import (
 )
 
 
-class YandexMusicBaseIE(InfoExtractor):
+class YandexMusicTrackIE(InfoExtractor):
+    IE_NAME = 'yandexmusic:track'
+    IE_DESC = 'Яндекс.Музыка - Трек'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://music.yandex.ru/album/540508/track/4878838',
+        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
+        'info_dict': {
+            'id': '4878838',
+            'ext': 'mp3',
+            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+            'filesize': 4628061,
+            'duration': 193.04,
+        }
+    }
+
     def _get_track_url(self, storage_dir, track_id):
         data = self._download_json(
             'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s'
@@ -35,24 +51,6 @@ class YandexMusicBaseIE(InfoExtractor):
             'duration': float_or_none(track.get('durationMs'), 1000),
         }
 
-
-class YandexMusicTrackIE(YandexMusicBaseIE):
-    IE_NAME = 'yandexmusic:track'
-    IE_DESC = 'Яндекс.Музыка - Трек'
-    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://music.yandex.ru/album/540508/track/4878838',
-        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
-        'info_dict': {
-            'id': '4878838',
-            'ext': 'mp3',
-            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
-            'filesize': 4628061,
-            'duration': 193.04,
-        }
-    }
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         album_id, track_id = mobj.group('album_id'), mobj.group('id')

From 11addc50ffa9ce65ac3bef7af6b1c38d7eae1af6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Aug 2015 23:52:47 +0200
Subject: [PATCH 1132/2145] release 2015.08.23

---
 docs/supportedsites.md | 6 +++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9099e2da4..8d9db53a6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -301,6 +301,7 @@
  - **Moviezine**
  - **movshare**: MovShare
  - **MPORA**
+ - **MSNBC**
  - **MTV**
  - **mtviggy.com**
  - **mtvservices:embedded**
@@ -308,6 +309,7 @@
  - **MusicPlayOn**
  - **MusicVault**
  - **muzu.tv**
+ - **Mwave**
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
@@ -392,6 +394,8 @@
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **Playwire**
+ - **pluralsight**
+ - **pluralsight:course**
  - **plus.google**: Google Plus
  - **pluzz.francetv.fr**
  - **podomatic**
@@ -534,6 +538,7 @@
  - **TF1**
  - **TheOnion**
  - **ThePlatform**
+ - **ThePlatformFeed**
  - **TheSixtyOne**
  - **ThisAmericanLife**
  - **ThisAV**
@@ -599,7 +604,6 @@
  - **Viddler**
  - **video.google:search**: Google Video search
  - **video.mit.edu**
- - **VideoBam**
  - **VideoDetective**
  - **videofy.me**
  - **videolectures.net**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c090c6df7..394951ca7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.16.1'
+__version__ = '2015.08.23'

From eba470f2f22389ab32164e4eb39067ceecf900f5 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 24 Aug 2015 16:30:00 +0800
Subject: [PATCH 1133/2145] [vlive] Remove upload_date extraction & cleanup

---
 youtube_dl/extractor/vlive.py | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index b3bbd80fb..6a403cc64 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -25,7 +25,6 @@ class VLiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': '[V] Girl\'s Day\'s Broadcast',
             'creator': 'Girl\'s Day',
-            'upload_date': '20150817',
         },
     }
     _SECRET = 'rFkwZet6pqk1vQt6SxxUkAHX7YL3lmqzUMrU4IDusTo4jEBdtOhNfT4BYYAdArwH'
@@ -41,21 +40,14 @@ class VLiveIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
             r'<span class="name">([^<>]+)</span>', webpage, 'creator')
-        upload_date = self._html_search_regex(
-            r'<span class="time">(\d{4}\.\d{2}\.\d{2})</span>', webpage,
-            'upload date', default=None, fatal=False)
-        if upload_date:
-            upload_date = upload_date.replace('.', '')
-
+        
         url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
-        msgpad = {'msgpad': '%.0f' % (time() * 1000)}
-        md = {
-            'md': b64encode(
-                hmac.new(self._SECRET.encode('ascii'),
-                         (url[:255] + msgpad['msgpad']).encode('ascii'), sha1).digest())
-        }
-        url += '&' + compat_urllib_parse.urlencode(msgpad) + '&' + compat_urllib_parse.urlencode(md)
-
+        msgpad = '%.0f' % (time() * 1000)
+        md = b64encode(
+            hmac.new(self._SECRET.encode('ascii'),
+                     (url[:255] + msgpad).encode('ascii'), sha1).digest()
+        )
+        url += '&' + compat_urllib_parse.urlencode({'msgpad': msgpad, 'md': md})
         playinfo = self._download_json(url, video_id, 'Downloading video json')
 
         if playinfo.get('message', '') != 'success':
@@ -89,6 +81,5 @@ class VLiveIE(InfoExtractor):
             'creator': creator,
             'thumbnail': thumbnail,
             'formats': formats,
-            'upload_date': upload_date,
             'subtitles': subtitles,
         }

From 95e431e9ec2477694d368a050222d6381a6f88ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Aug 2015 21:08:38 +0600
Subject: [PATCH 1134/2145] [mailru] Skip tests

---
 youtube_dl/extractor/mailru.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 54a14cb94..ab1300185 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -25,6 +25,7 @@ class MailRuIE(InfoExtractor):
                 'uploader_id': 'sonypicturesrus@mail.ru',
                 'duration': 184,
             },
+            'skip': 'Not accessible from Travis CI server',
         },
         {
             'url': 'http://my.mail.ru/corp/hitech/video/news_hi-tech_mail_ru/1263.html',
@@ -39,6 +40,7 @@ class MailRuIE(InfoExtractor):
                 'uploader_id': 'hitech@corp.mail.ru',
                 'duration': 245,
             },
+            'skip': 'Not accessible from Travis CI server',
         },
     ]
 

From ebbf078c7df575903ceb1be53e53533508c79dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Aug 2015 21:19:21 +0600
Subject: [PATCH 1135/2145] [krasview] Skip download for test

---
 youtube_dl/extractor/krasview.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index 96f95979a..0ae8ebd68 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -25,6 +25,9 @@ class KrasViewIE(InfoExtractor):
             'duration': 27,
             'thumbnail': 're:^https?://.*\.jpg',
         },
+        'params': {
+            'skip_download': 'Not accessible from Travis CI server',
+        },
     }
 
     def _real_extract(self, url):

From 6d53cdd6ce441dd7bc1d93bf1445f0594cfdffef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Aug 2015 23:29:02 +0600
Subject: [PATCH 1136/2145] [yandexmusic] Skip removed tracks (#6666)

---
 youtube_dl/extractor/yandexmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 91829be1c..166cbf344 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -67,7 +67,7 @@ class YandexMusicPlaylistBaseIE(InfoExtractor):
         return [
             self.url_result(
                 'http://music.yandex.ru/album/%s/track/%s' % (track['albums'][0]['id'], track['id']))
-            for track in tracks]
+            for track in tracks if track.get('albums') and isinstance(track.get('albums'), list)]
 
 
 class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):

From baf510bf8cb296d2ed2a2f742ec9387d094623e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 00:11:15 +0600
Subject: [PATCH 1137/2145] [yandexmusic:playlist] Handle playlists with more
 than 150 tracks (Closes #6666)

---
 youtube_dl/extractor/yandexmusic.py | 51 +++++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 166cbf344..4098e4629 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -5,7 +5,11 @@ import re
 import hashlib
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
 from ..utils import (
     int_or_none,
     float_or_none,
@@ -106,7 +110,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
     IE_DESC = 'Яндекс.Музыка - Плейлист'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
         'info_dict': {
             'id': '1245',
@@ -114,19 +118,54 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
         },
         'playlist_count': 6,
-    }
+    }, {
+        # playlist exceeding the limit of 150 tracks shipped with webpage (see
+        # https://github.com/rg3/youtube-dl/issues/6666)
+        'url': 'https://music.yandex.ru/users/ya.playlist/playlists/1036',
+        'info_dict': {
+            'id': '1036',
+            'title': 'Музыка 90-х',
+        },
+        'playlist_count': 310,
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
-        playlist = self._parse_json(
+        mu = self._parse_json(
             self._search_regex(
                 r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
-            playlist_id)['pageData']['playlist']
+            playlist_id)
+
+        playlist = mu['pageData']['playlist']
+        tracks, track_ids = playlist['tracks'], playlist['trackIds']
+
+        # tracks dictionary shipped with webpage is limited to 150 tracks,
+        # missing tracks should be retrieved manually.
+        if len(tracks) < len(track_ids):
+            present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
+            missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
+            request = compat_urllib_request.Request(
+                'https://music.yandex.ru/handlers/track-entries.jsx',
+                compat_urllib_parse.urlencode({
+                    'entries': ','.join(missing_track_ids),
+                    'lang': mu.get('settings', {}).get('lang', 'en'),
+                    'external-domain': 'music.yandex.ru',
+                    'overembed': 'false',
+                    'sign': mu.get('authData', {}).get('user', {}).get('sign'),
+                    'strict': 'true',
+                }).encode('utf-8'))
+            request.add_header('Referer', url)
+            request.add_header('X-Requested-With', 'XMLHttpRequest')
+
+            missing_tracks = self._download_json(
+                request, playlist_id, 'Downloading missing tracks JSON', fatal=False)
+            if missing_tracks:
+                tracks.extend(missing_tracks)
 
         return self.playlist_result(
-            self._build_playlist(playlist['tracks']),
+            self._build_playlist(tracks),
             compat_str(playlist_id),
             playlist['title'], playlist.get('description'))

From 4bc8eec4ebf5ffcca3b2e17c864be08df5215f44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 26 Aug 2015 15:21:55 +0200
Subject: [PATCH 1138/2145] [youtube] Adapt player version regex to handle urls
 ending in '/html5player-new.js'

It was always extracting 'new' as the version, breaking the cache system.
---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8e2da46e3..ab6754154 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -660,7 +660,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)
@@ -1289,7 +1289,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 player_desc = 'flash player %s' % player_version
                             else:
                                 player_version = self._search_regex(
-                                    r'html5player-([^/]+?)(?:/html5player)?\.js',
+                                    r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From 2f72e83bbd915054cac0e8f70df0c2cab4b9c116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 20:47:57 +0600
Subject: [PATCH 1139/2145] [crunchyroll] Detect required login (#6677)

---
 youtube_dl/extractor/crunchyroll.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 33a033a7f..98d1881ae 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -237,7 +237,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             webpage_url = 'http://www.' + mobj.group('url')
 
         webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
-        note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='')
+        note_m = self._html_search_regex(
+            r'<div class="showmedia-trailer-notice">(.+?)</div>',
+            webpage, 'trailer-notice', default='')
         if note_m:
             raise ExtractorError(note_m)
 
@@ -247,6 +249,12 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             if msg.get('type') == 'error':
                 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
 
+        if 'To view this, please log in to verify you are 18 or older.' in webpage:
+            raise ExtractorError(
+                'This video is only available for registered users, '
+                'use --username and --password options to provide account credentials.',
+                expected=True)
+
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')

From 43e7d3c9453338ae29552311b1447fe95be05db5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:24:47 +0600
Subject: [PATCH 1140/2145] [extractor/common] Add raise_login_required

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5d24bcb6a..39cef9c5b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -510,6 +510,12 @@ class InfoExtractor(object):
         """Report attempt to log in."""
         self.to_screen('Logging in')
 
+    @staticmethod
+    def raise_login_required(msg='This video is only available for registered users'):
+        raise ExtractorError(
+            '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
+            expected=True)
+
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):

From bbb43a39fd11c2fdf28ae593eaa994f22ce663bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:25:04 +0600
Subject: [PATCH 1141/2145] [crunchyroll] Use raise_login_required

---
 youtube_dl/extractor/crunchyroll.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 98d1881ae..801b9b48e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -250,10 +250,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
 
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
-            raise ExtractorError(
-                'This video is only available for registered users, '
-                'use --username and --password options to provide account credentials.',
-                expected=True)
+            self.raise_login_required(video_id)
 
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)

From 3c53455d15035a94bcd2bc915f565420e1a4279f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:25:37 +0600
Subject: [PATCH 1142/2145] [eroprofile] Use raise_login_required

---
 youtube_dl/extractor/eroprofile.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 316033cf1..7fcd0151d 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -71,8 +71,7 @@ class EroProfileIE(InfoExtractor):
 
         m = re.search(r'You must be logged in to view this video\.', webpage)
         if m:
-            raise ExtractorError(
-                'This video requires login. Please specify a username and password and try again.', expected=True)
+            self.raise_login_required('This video requires login')
 
         video_id = self._search_regex(
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],

From 62984e4584c2962e622514c7d6a475636a8c21d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:25:53 +0600
Subject: [PATCH 1143/2145] [lynda] Use raise_login_required

---
 youtube_dl/extractor/lynda.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5b9157ed4..378117270 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -118,9 +118,7 @@ class LyndaIE(LyndaBaseIE):
                 'lynda returned error: %s' % video_json['Message'], expected=True)
 
         if video_json['HasAccess'] is False:
-            raise ExtractorError(
-                'Video %s is only available for members. '
-                % video_id + self._ACCOUNT_CREDENTIALS_HINT, expected=True)
+            self.raise_login_required('Video %s is only available for members' % video_id)
 
         video_id = compat_str(video_json['ID'])
         duration = video_json['DurationInSeconds']

From e7ddaef5bd209dd8d24b0025631cde1f5969e71d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:09 +0600
Subject: [PATCH 1144/2145] [pluralsight] Use raise_login_required

---
 youtube_dl/extractor/pluralsight.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7ba396aef..fd32836cc 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -41,9 +41,7 @@ class PluralsightIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            raise ExtractorError(
-                'Pluralsight account is required, use --username and --password options to provide account credentials.',
-                expected=True)
+            self.raise_login_required('Pluralsight account is required')
 
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')

From e269d3ae7dbebb22d5b51bd5e6d477a69ae4f3b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:24 +0600
Subject: [PATCH 1145/2145] [safari] Use raise_login_required

---
 youtube_dl/extractor/safari.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index f3c80708c..a602af692 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -20,7 +20,6 @@ from ..utils import (
 class SafariBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
     _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
-    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to supply credentials for safaribooksonline.com'
     _NETRC_MACHINE = 'safari'
 
     _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
@@ -37,9 +36,7 @@ class SafariBaseIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            raise ExtractorError(
-                self._ACCOUNT_CREDENTIALS_HINT,
-                expected=True)
+            self.raise_login_required('safaribooksonline.com account is required')
 
         headers = std_headers
         if 'Referer' not in headers:

From 42e7373bd3c819ee7cebf5898e4bdd33730dde6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:35 +0600
Subject: [PATCH 1146/2145] [smotri] Use raise_login_required

---
 youtube_dl/extractor/smotri.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 93a7cfe15..35a81ee87 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -330,10 +330,7 @@ class SmotriBroadcastIE(InfoExtractor):
 
             (username, password) = self._get_login_info()
             if username is None:
-                raise ExtractorError(
-                    'Erotic broadcasts allowed only for registered users, '
-                    'use --username and --password options to provide account credentials.',
-                    expected=True)
+                self.raise_login_required('Erotic broadcasts allowed only for registered users')
 
             login_form = {
                 'login-hint53': '1',

From 61a7ff16222accdb259f771d0a6f0adb229b34dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:47 +0600
Subject: [PATCH 1147/2145] [tubitv] Use raise_login_required

---
 youtube_dl/extractor/tubitv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 2c4b21807..4f86b3ee9 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -60,9 +60,7 @@ class TubiTvIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         if re.search(r"<(?:DIV|div) class='login-required-screen'>", webpage):
-            raise ExtractorError(
-                'This video requires login, use --username and --password '
-                'options to provide account credentials.', expected=True)
+            self.raise_login_required('This video requires login')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)

From a882c5f4747c527bb50d87828ea4cceae6d12533 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:27:07 +0600
Subject: [PATCH 1148/2145] [udemy] Use raise_login_required

---
 youtube_dl/extractor/udemy.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 4a0eaf65f..365d8b4bf 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -70,9 +70,7 @@ class UdemyIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            raise ExtractorError(
-                'Udemy account is required, use --username and --password options to provide account credentials.',
-                expected=True)
+            self.raise_login_required('Udemy account is required')
 
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')

From 39affb5aa427a3a1830e97523470d11bfdbd067e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:27:57 +0600
Subject: [PATCH 1149/2145] [crunchyroll] Fix typo

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 801b9b48e..c2162aa68 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -250,7 +250,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
 
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
-            self.raise_login_required(video_id)
+            self.raise_login_required()
 
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)

From 3d8132f5e20b7cbdaa8f69aca482553b2c02bed8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:03:58 +0600
Subject: [PATCH 1150/2145] [shared] Extend _VALID_URL to support vivo.sx
 (Closes #6681)

---
 youtube_dl/extractor/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index a07677686..000ef1a07 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class SharedIE(InfoExtractor):
-    _VALID_URL = r'http://shared\.sx/(?P<id>[\da-z]{10})'
+    _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
     _TEST = {
         'url': 'http://shared.sx/0060718775',

From 70113c38c9e551d7d9ab2a4d1f7e76c81b68ae76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:04:39 +0600
Subject: [PATCH 1151/2145] [shared] Clarify IE_DESC

---
 youtube_dl/extractor/shared.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 000ef1a07..cf0a3bfef 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class SharedIE(InfoExtractor):
+    IE_DESC = 'shared.sx and vivo.sx'
     _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
     _TEST = {

From f62e02c24f1f0e0488b40df178ddb9bb5fdf9fc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:05:45 +0600
Subject: [PATCH 1152/2145] [shared] Add test for vivo

---
 youtube_dl/extractor/shared.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index cf0a3bfef..4fa991dff 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -17,7 +17,7 @@ class SharedIE(InfoExtractor):
     IE_DESC = 'shared.sx and vivo.sx'
     _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://shared.sx/0060718775',
         'md5': '106fefed92a8a2adb8c98e6a0652f49b',
         'info_dict': {
@@ -25,7 +25,16 @@ class SharedIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Bmp4',
         },
-    }
+    }, {
+        'url': 'http://vivo.sx/d7ddda0e78',
+        'md5': '15b3af41be0b4fe01f4df075c2678b2c',
+        'info_dict': {
+            'id': 'd7ddda0e78',
+            'ext': 'mp4',
+            'title': 'Chicken',
+            'filesize': 528031,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From f11c316347bea41d9148d1c8d5d7738a594a06d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:06:10 +0600
Subject: [PATCH 1153/2145] [shared] Add filesize to test

---
 youtube_dl/extractor/shared.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 4fa991dff..c5636e8e9 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -24,6 +24,7 @@ class SharedIE(InfoExtractor):
             'id': '0060718775',
             'ext': 'mp4',
             'title': 'Bmp4',
+            'filesize': 1720110,
         },
     }, {
         'url': 'http://vivo.sx/d7ddda0e78',

From d7e8264517d29156697f82b7761dc99d13994c21 Mon Sep 17 00:00:00 2001
From: nmrugg <nmrugg@gmail.com>
Date: Thu, 27 Aug 2015 23:24:13 +0800
Subject: [PATCH 1154/2145] Make FoxBusiness work.

---
 youtube_dl/extractor/foxnews.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 917f76b1e..7de88ab66 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
@@ -8,7 +10,7 @@ from ..utils import (
 
 
 class FoxNewsIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.foxnews\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _VALID_URL = r'https?://video\.fox(?:news|business)\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
@@ -47,8 +49,10 @@ class FoxNewsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        m = re.match(r'^https?://video\.fox(news|business)', url)
+
         video = self._download_json(
-            'http://video.foxnews.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
+            'http://video.fox' + m.group(1) + '.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
 
         item = video['channel']['item']
         title = item['title']

From 8df8c278b6d5e2b5a350446690873dc9f5f48aff Mon Sep 17 00:00:00 2001
From: nmrugg <nmrugg@gmail.com>
Date: Thu, 27 Aug 2015 23:24:28 +0800
Subject: [PATCH 1155/2145] Added matching test for FoxBusiness.

---
 youtube_dl/extractor/foxnews.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 7de88ab66..a8902c960 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -44,6 +44,10 @@ class FoxNewsIE(InfoExtractor):
             'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',
             'only_matching': True,
         },
+        {
+            'url': 'http://video.foxbusiness.com/v/4442309889001',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 1b660cce120c733f2bb195ef1cfe2ff2421b439f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Aug 2015 21:48:03 +0600
Subject: [PATCH 1156/2145] [foxnews] Simplify (Closes #6694)

---
 youtube_dl/extractor/foxnews.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index a8902c960..244c75f0b 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class FoxNewsIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.fox(?:news|business)\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
@@ -51,12 +51,12 @@ class FoxNewsIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        m = re.match(r'^https?://video\.fox(news|business)', url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
 
         video = self._download_json(
-            'http://video.fox' + m.group(1) + '.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
+            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id), video_id)
 
         item = video['channel']['item']
         title = item['title']

From 5307c332329d6a1f3eec240b66a4f11905889f23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Aug 2015 21:48:47 +0600
Subject: [PATCH 1157/2145] [foxnews] Clarify IE_DESC

---
 youtube_dl/extractor/foxnews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 244c75f0b..3a4a59135 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class FoxNewsIE(InfoExtractor):
+    IE_DESC = 'Fox News and Fox Business Video'
     _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {

From a4962b80d668de704fc347d5e76587be0e95dfef Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 28 Aug 2015 05:04:39 +0200
Subject: [PATCH 1158/2145] release 2015.08.28

---
 docs/supportedsites.md | 4 ++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8d9db53a6..328a819b3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -166,7 +166,7 @@
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
  - **Foxgay**
- - **FoxNews**
+ - **FoxNews**: Fox News and Fox Business Video
  - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
@@ -465,7 +465,7 @@
  - **Sexu**
  - **SexyKarma**: Sexy Karma and Watch Indian Porn
  - **Shahid**
- - **Shared**
+ - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
  - **Sina**
  - **Slideshare**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 394951ca7..a07bc9233 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.23'
+__version__ = '2015.08.28'

From 071c10137b6b17b79ecfc8676736d5cc243022f6 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 26 Aug 2015 00:06:44 +0200
Subject: [PATCH 1159/2145] [MTV] move German mtv site to new class

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mtv.py      | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d59882598..66422b005 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -340,6 +340,7 @@ from .mtv import (
     MTVIE,
     MTVServicesEmbeddedIE,
     MTVIggyIE,
+    MTVDEIE,
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index b48fac5e3..15df62649 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -288,3 +288,40 @@ class MTVIggyIE(MTVServicesInfoExtractor):
         }
     }
     _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'
+
+class MTVDEIE(MTVServicesInfoExtractor):
+    IE_NAME = 'mtv.de'
+    _VALID_URL = r'''(?x)^https?://(?:www\.)?mtv\.de(?P<video_path>/artists/.*)'''
+    _TESTS = [
+        {
+            'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
+            'info_dict': {
+                'id': 'a50bc5f0b3aa4b3190aa',
+                'ext': 'mp4',
+                'title': 'cro-traum',
+                'description': 'Cro - Traum',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        return self._get_videos_info(url, mobj.group('video_path'))
+
+    def _get_videos_info(self, url, video_path):
+        webpage = self._download_webpage(url, video_path)
+        playlist_js = self._search_regex(r'<script>\s*window.pagePlaylist =(.*?\]);\s*window.trackingParams =', webpage, 'playlist', flags=re.DOTALL)
+        playlist = self._parse_json(playlist_js, video_path)
+        info = None
+        for item in playlist:
+            if item['video_path'] == video_path:
+                info = item
+                break
+        if info == None:
+            raise ExtractorError('video not in playlist')
+        mrss_url = info['mrss']
+        idoc = self._download_xml(
+            mrss_url, video_path,
+            'Downloading info', transform_source=fix_xml_ampersands)
+        return self.playlist_result(
+            [self._get_video_info(item) for item in idoc.findall('.//item')])

From 79fa9db0da4aa86aa101cc32a5da4b7e254c6a0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:24:54 +0600
Subject: [PATCH 1160/2145] [mtvde] Simplify (Closes #6673)

---
 youtube_dl/extractor/mtv.py | 59 ++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 15df62649..f2d0638bf 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -174,8 +174,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if self._LANG:
             info_url += 'lang=%s&' % self._LANG
         info_url += data
+        return self._get_videos_info_from_url(info_url, video_id)
+
+    def _get_videos_info_from_url(self, url, video_id):
         idoc = self._download_xml(
-            info_url, video_id,
+            url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
         return self.playlist_result(
             [self._get_video_info(item) for item in idoc.findall('.//item')])
@@ -289,39 +292,35 @@ class MTVIggyIE(MTVServicesInfoExtractor):
     }
     _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'
 
+
 class MTVDEIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv.de'
-    _VALID_URL = r'''(?x)^https?://(?:www\.)?mtv\.de(?P<video_path>/artists/.*)'''
-    _TESTS = [
-        {
-            'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
-            'info_dict': {
-                'id': 'a50bc5f0b3aa4b3190aa',
-                'ext': 'mp4',
-                'title': 'cro-traum',
-                'description': 'Cro - Traum',
-            },
+    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows)/(?:[^/]+/)+(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
+    _TESTS = [{
+        'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
+        'info_dict': {
+            'id': 'music_video-a50bc5f0b3aa4b3190aa',
+            'ext': 'mp4',
+            'title': 'MusicVideo_cro-traum',
+            'description': 'Cro - Traum',
         },
-    ]
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        return self._get_videos_info(url, mobj.group('video_path'))
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'window\.pagePlaylist\s*=\s*(\[.+?\]);\n', webpage, 'page playlist'),
+            video_id)
 
-    def _get_videos_info(self, url, video_path):
-        webpage = self._download_webpage(url, video_path)
-        playlist_js = self._search_regex(r'<script>\s*window.pagePlaylist =(.*?\]);\s*window.trackingParams =', webpage, 'playlist', flags=re.DOTALL)
-        playlist = self._parse_json(playlist_js, video_path)
-        info = None
         for item in playlist:
-            if item['video_path'] == video_path:
-                info = item
-                break
-        if info == None:
-            raise ExtractorError('video not in playlist')
-        mrss_url = info['mrss']
-        idoc = self._download_xml(
-            mrss_url, video_path,
-            'Downloading info', transform_source=fix_xml_ampersands)
-        return self.playlist_result(
-            [self._get_video_info(item) for item in idoc.findall('.//item')])
+            item_id = item.get('id')
+            if item_id and compat_str(item_id) == video_id:
+                return self._get_videos_info_from_url(item['mrss'], video_id)

From 56f447be9fa196bceafbacb6526cad7c122a30ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:27:36 +0600
Subject: [PATCH 1161/2145] [mtvservices] Fix mediagen URL generation

---
 youtube_dl/extractor/mtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index f2d0638bf..56b255599 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -114,7 +114,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
         # Remove the templates, like &device={device}
         mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
         if 'acceptMethods' not in mediagen_url:
-            mediagen_url += '&acceptMethods=fms'
+            mediagen_url += '&' if '?' in mediagen_url else '?'
+            mediagen_url += 'acceptMethods=fms'
 
         mediagen_doc = self._download_xml(mediagen_url, video_id,
                                           'Downloading video urls')

From c3c9f879541b99a5456991d887ed03a2aea5dcff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:31:38 +0600
Subject: [PATCH 1162/2145] [mtvde] Add test for mediagen URL without query

---
 youtube_dl/extractor/mtv.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 56b255599..28301d406 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -309,6 +309,18 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        # mediagen URL without query (e.g. http://videos.mtvnn.com/mediagen/e865da714c166d18d6f80893195fcb97)
+        'url': 'http://www.mtv.de/shows/933-teen-mom-2/staffeln/5353/folgen/63565-enthullungen',
+        'info_dict': {
+            'id': 'local_playlist-f5ae778b9832cc837189',
+            'ext': 'mp4',
+            'title': 'Episode_teen-mom-2_shows_season-5_episode-1_full-episode_part1',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 65488b820c6792f6c9f5e729c417985d501c8eb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:41:54 +0600
Subject: [PATCH 1163/2145] [mtvde] Add support for news

---
 youtube_dl/extractor/mtv.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 28301d406..d160f4a22 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -296,7 +296,7 @@ class MTVIggyIE(MTVServicesInfoExtractor):
 
 class MTVDEIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv.de'
-    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows)/(?:[^/]+/)+(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows|news)/(?:[^/]+/)*(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
     _TESTS = [{
         'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
         'info_dict': {
@@ -321,6 +321,19 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        # single video in pagePlaylist with different id
+        'url': 'http://www.mtv.de/news/77491-mtv-movies-spotlight-pixels-teil-3',
+        'info_dict': {
+            'id': 'local_playlist-4e760566473c4c8c5344',
+            'ext': 'mp4',
+            'title': 'Article_mtv-movies-spotlight-pixels-teil-3_short-clips_part1',
+            'description': 'MTV Movies Supercut',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -333,6 +346,10 @@ class MTVDEIE(MTVServicesInfoExtractor):
                 r'window\.pagePlaylist\s*=\s*(\[.+?\]);\n', webpage, 'page playlist'),
             video_id)
 
+        # news pages contain single video in playlist with different id
+        if len(playlist) == 1:
+            return self._get_videos_info_from_url(playlist[0]['mrss'], video_id)
+
         for item in playlist:
             item_id = item.get('id')
             if item_id and compat_str(item_id) == video_id:

From 054d43bb11590da234e7e8f46a595a34b19efb8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:45:29 +0600
Subject: [PATCH 1164/2145] [mtvservices] Catch more georestriction errors

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index d160f4a22..4e4358151 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -67,7 +67,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return [{'url': url, 'ext': 'mp4'}]
 
     def _extract_video_formats(self, mdoc, mtvn_id):
-        if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4)$', mdoc.find('.//src').text) is not None:
+        if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4|copyright_error\.flv(?:\?geo\b.+?)?)$', mdoc.find('.//src').text) is not None:
             if mtvn_id is not None and self._MOBILE_TEMPLATE is not None:
                 self.to_screen('The normal version is not available from your '
                                'country, trying with the mobile version')

From 20149a5da195b8ab5e50a4babfa0107ebf279b7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 22:10:49 +0600
Subject: [PATCH 1165/2145] [mtvservices] Fix title extraction under python 2

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 4e4358151..a597714e9 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -142,7 +142,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if title_el is None:
             title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
         if title_el is None:
-            title_el = itemdoc.find('.//title')
+            title_el = itemdoc.find('.//title') or itemdoc.find('./title')
             if title_el.text is None:
                 title_el = None
 

From 8ca31a0e056a7a7b71ea35753504ed0f870e834f Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 29 Aug 2015 07:55:20 +0200
Subject: [PATCH 1166/2145] [generic] Look for ScreenwaveMedia embeds

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 376feecae..5d41b6319 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1718,6 +1718,15 @@ class GenericIE(InfoExtractor):
         if snagfilms_url:
             return self.url_result(snagfilms_url)
 
+        # Look for ScreenwaveMedia embeds
+        mobj = re.search(
+            r'<script[^>]+src=[\'"](http://player\d?\.screenwavemedia\.com/player\.php\?id=[A-Za-z0-9-]+)[\'"]',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                unescapeHTML(mobj.group(1)),
+                'ScreenwaveMedia')
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 9b22cb10c486b827102093d05abeb21a4d4913b7 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 29 Aug 2015 08:00:25 +0200
Subject: [PATCH 1167/2145] [screenwavemedia] use the true JS value instead of
 a comment

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 220d39078..e1604b5c0 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -33,7 +33,7 @@ class ScreenwaveMediaIE(InfoExtractor):
             'http://player.screenwavemedia.com/player.js',
             video_id, 'Downloading playerconfig webpage')
 
-        videoserver = self._search_regex(r"\[ipaddress\]\s*=>\s*([\d\.]+)", playerdata, 'videoserver')
+        videoserver = self._search_regex(r'SWMServer\s*=\s*"([\d\.]+)"', playerdata, 'videoserver')
 
         sources = self._parse_json(
             js_to_json(

From e276fd2cb39a94b1f40f47a5c52913aac8557976 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 29 Aug 2015 08:01:10 +0200
Subject: [PATCH 1168/2145] [screenwavemedia] warn when falling back to a
 hardcoded list of streams

---
 youtube_dl/extractor/screenwavemedia.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index e1604b5c0..671ea57d0 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -56,6 +56,7 @@ class ScreenwaveMediaIE(InfoExtractor):
 
         # Fallback to hardcoded sources if JS changes again
         if not sources:
+            self.report_warning('Falling back to a hardcoded list of streams')
             sources = [{
                 'file': 'http://%s/vod/%s_%s.mp4' % (videoserver, video_id, format_id),
                 'type': 'mp4',

From 5e58956d0a057e7294af15bef5a22c675812a18b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 18:57:44 +0600
Subject: [PATCH 1169/2145] [screenwavemedia] Add EMBED_PATTERN

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 671ea57d0..4d1379615 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class ScreenwaveMediaIE(InfoExtractor):
     _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
-
+    EMBED_PATTERN = r'src=(["\'])(?P<url>(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=.+?)\1'
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
         'only_matching': True,

From 678d33295ba45833cedc2abab9c906d12dae9bee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:05:11 +0600
Subject: [PATCH 1170/2145] [screenwavemedia] Improve _VALID_URL

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 4d1379615..05f93904c 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
+    _VALID_URL = r'https?://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=(?P<id>[A-Za-z0-9-]+)'
     EMBED_PATTERN = r'src=(["\'])(?P<url>(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=.+?)\1'
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',

From 109a4156e17b2c0dee414151fa37db2d208df43d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:06:55 +0600
Subject: [PATCH 1171/2145] [cinemassacre] Use screenwavemedia embed pattern

---
 youtube_dl/extractor/cinemassacre.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index c949a4814..fd1770dac 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import ExtractorError
 from .bliptv import BlipTVIE
+from .screenwavemedia import ScreenwaveMediaIE
 
 
 class CinemassacreIE(InfoExtractor):
@@ -83,10 +84,10 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             [
-                r'src="(http://(?:player2\.screenwavemedia\.com|player\.screenwavemedia\.com/play)/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
-                r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
+                ScreenwaveMediaIE.EMBED_PATTERN,
+                r'<iframe[^>]+src="(?P<url>(?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
-            webpage, 'player data URL', default=None)
+            webpage, 'player data URL', default=None, group='url')
         if not playerdata_url:
             playerdata_url = BlipTVIE._extract_url(webpage)
         if not playerdata_url:

From efd712c69b6c77d216c1ec3e6243e49254f5e37f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:07:31 +0600
Subject: [PATCH 1172/2145] [generic] Use screenwavemedia embed pattern

---
 youtube_dl/extractor/generic.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5d41b6319..c9b36221e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,7 @@ from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
+from .screenwavemedia import ScreenwaveMediaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1719,13 +1720,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(snagfilms_url)
 
         # Look for ScreenwaveMedia embeds
-        mobj = re.search(
-            r'<script[^>]+src=[\'"](http://player\d?\.screenwavemedia\.com/player\.php\?id=[A-Za-z0-9-]+)[\'"]',
-            webpage)
+        mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
         if mobj is not None:
-            return self.url_result(
-                unescapeHTML(mobj.group(1)),
-                'ScreenwaveMedia')
+            return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
 
         # Look for AdobeTVVideo embeds
         mobj = re.search(

From 1f8125805e7af127efd331a41775b7b5c90a77c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:12:38 +0600
Subject: [PATCH 1173/2145] [generic] Add test for screenwavemedia embed

---
 youtube_dl/extractor/generic.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c9b36221e..953ec32c3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1002,6 +1002,16 @@ class GenericIE(InfoExtractor):
                 'description': 'New experience with Acrobat DC',
                 'duration': 248.667,
             },
+        },
+        # ScreenwaveMedia embed
+        {
+            'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
+            'md5': '24ace5baba0d35d55c6810b51f34e9e0',
+            'info_dict': {
+                'id': 'cinemasnob-55d26273809dd',
+                'ext': 'mp4',
+                'title': 'cinemasnob',
+            },
         }
     ]
 

From ef49b5905345255bc6913e279b9d76a0bb83e1ce Mon Sep 17 00:00:00 2001
From: sourcerect <hypershadsy@gmail.com>
Date: Sat, 29 Aug 2015 02:12:15 -0400
Subject: [PATCH 1174/2145] [kaltura] add html5 player urls

---
 youtube_dl/extractor/kaltura.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index d28730492..1f64df4d5 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -14,11 +14,12 @@ from ..utils import (
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
     (?:kaltura:|
-       https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
-    )(?P<partner_id>\d+)
-    (?::|
-       /(?:[^/]+/)*?entry_id/
-    )(?P<id>[0-9a-z_]+)'''
+        https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/(?:
+            (?:index\.php/kwidget/(?:[^/]+/)*?wid/_)|
+            (?:html5/html5lib/v(?:[\d.]+)/mwEmbedFrame.php/p/\d+)
+        )
+    )(?P<partner_id>\d+)?(?::|/(?:[^/]+/)*?entry_id/)(?P<id>[0-9a-z_]+)
+    (?:\?wid=_(?P<partner_id_html5>\d+))?'''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
     _TESTS = [
         {
@@ -43,6 +44,10 @@ class KalturaIE(InfoExtractor):
             'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3',
             'only_matching': True,
         },
+        {
+            'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.30.2/mwEmbedFrame.php/p/1337/uiconf_id/20540612/entry_id/1_sf5ovm7u?wid=_243342',
+            'only_matching': True,
+        }
     ]
 
     def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
@@ -107,7 +112,7 @@ class KalturaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         mobj = re.match(self._VALID_URL, url)
-        partner_id, entry_id = mobj.group('partner_id'), mobj.group('id')
+        partner_id, entry_id = mobj.group('partner_id') or mobj.group('partner_id_html5'), mobj.group('id')
 
         info, source_data = self._get_video_info(entry_id, partner_id)
 

From ee3ec091f46bd52979274d0aa42ff1f1c3344de6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 20:21:59 +0600
Subject: [PATCH 1175/2145] [kaltura] Strictify _VALID_URL

---
 youtube_dl/extractor/kaltura.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 1f64df4d5..c57f33ece 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -13,13 +13,25 @@ from ..utils import (
 
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-    (?:kaltura:|
-        https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/(?:
-            (?:index\.php/kwidget/(?:[^/]+/)*?wid/_)|
-            (?:html5/html5lib/v(?:[\d.]+)/mwEmbedFrame.php/p/\d+)
-        )
-    )(?P<partner_id>\d+)?(?::|/(?:[^/]+/)*?entry_id/)(?P<id>[0-9a-z_]+)
-    (?:\?wid=_(?P<partner_id_html5>\d+))?'''
+                (?:
+                    kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
+                    https?://
+                    (:?
+                        (?:www|cdnapisec)\.)?kaltura\.com/
+                        (?:
+                            (?:
+                                # flash player
+                                index\.php/kwidget/
+                                (?:[^/]+/)*?wid/_(?P<partner_id>\d+)/
+                                (?:[^/]+/)*?entry_id/(?P<id>[0-9a-z_]+)|
+                                # html player
+                                html5/html5lib/
+                                (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
+                                .*\?.*\bwid=_(?P<partner_id_html5>\d+)
+                        )
+                    )
+                )
+                '''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
     _TESTS = [
         {
@@ -110,9 +122,9 @@ class KalturaIE(InfoExtractor):
             video_id, actions, note='Downloading video info JSON')
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
         mobj = re.match(self._VALID_URL, url)
-        partner_id, entry_id = mobj.group('partner_id') or mobj.group('partner_id_html5'), mobj.group('id')
+        partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
+        entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
 
         info, source_data = self._get_video_info(entry_id, partner_id)
 
@@ -131,7 +143,7 @@ class KalturaIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': entry_id,
             'title': info['name'],
             'formats': formats,
             'description': info.get('description'),

From b184f94413ea2837a52816e3beb009f386351d70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 20:23:06 +0600
Subject: [PATCH 1176/2145] [kaltura] Clarify html5 player comment

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c57f33ece..c5b981092 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -24,7 +24,7 @@ class KalturaIE(InfoExtractor):
                                 index\.php/kwidget/
                                 (?:[^/]+/)*?wid/_(?P<partner_id>\d+)/
                                 (?:[^/]+/)*?entry_id/(?P<id>[0-9a-z_]+)|
-                                # html player
+                                # html5 player
                                 html5/html5lib/
                                 (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
                                 .*\?.*\bwid=_(?P<partner_id_html5>\d+)

From ae6a802106663a5bb0f94803dec21bf5feaa2527 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 20:26:51 +0600
Subject: [PATCH 1177/2145] [kaltura] Fix _VALID_URL formatting

---
 youtube_dl/extractor/kaltura.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c5b981092..3dca0e566 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -16,8 +16,7 @@ class KalturaIE(InfoExtractor):
                 (?:
                     kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
                     https?://
-                    (:?
-                        (?:www|cdnapisec)\.)?kaltura\.com/
+                        (:?(?:www|cdnapisec)\.)?kaltura\.com/
                         (?:
                             (?:
                                 # flash player
@@ -28,8 +27,8 @@ class KalturaIE(InfoExtractor):
                                 html5/html5lib/
                                 (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
                                 .*\?.*\bwid=_(?P<partner_id_html5>\d+)
+                            )
                         )
-                    )
                 )
                 '''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'

From c88e118b3cfb611bf2eae300a13885772481134f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 22:41:49 +0600
Subject: [PATCH 1178/2145] [nowtv] Fix display_id (Closes #6702)

---
 youtube_dl/extractor/nowtv.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 66c627bec..c8257719f 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -130,10 +130,16 @@ class NowTVIE(InfoExtractor):
     }, {
         'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        display_id_split = display_id.split('/')
+        if len(display_id) > 2:
+            display_id = '/'.join((display_id_split[0], display_id_split[-1]))
 
         info = self._download_json(
             'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,

From 17a647630b504c730d77e71faf248628e45529c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 30 Aug 2015 01:08:55 +0800
Subject: [PATCH 1179/2145] [abc] Support YouTube embeds (fixes #6595)

---
 youtube_dl/extractor/abc.py | 42 ++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index dc0fb85d6..06acf50d8 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -1,16 +1,19 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+)
 
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
     _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
         'md5': 'cb3dd03b18455a661071ee1e28344d9f',
         'info_dict': {
@@ -19,16 +22,41 @@ class ABCIE(InfoExtractor):
             'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
             'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
         },
-    }
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
+        'md5': 'db2a5369238b51f9811ad815b69dc086',
+        'info_dict': {
+            'id': 'NvqvPeNZsHU',
+            'ext': 'mp4',
+            'upload_date': '20150816',
+            'uploader': 'ABC News (Australia)',
+            'description': 'Government backbencher Warren Entsch introduces a cross-party sponsored bill to legalise same-sex marriage, saying the bill is designed to promote "an inclusive Australia, not a divided one.". Read more here: http://ab.co/1Mwc6ef',
+            'uploader_id': 'NewsOnABC',
+            'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
+        },
+        'add_ie': ['Youtube'],
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        urls_info_json = self._search_regex(
-            r'inlineVideoData\.push\((.*?)\);', webpage, 'video urls',
-            flags=re.DOTALL)
-        urls_info = json.loads(urls_info_json.replace('\'', '"'))
+        mobj = re.search(
+            r'inline(?P<type>Video|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
+            webpage)
+        if mobj is None:
+            raise ExtractorError('Unable to extract video urls')
+
+        urls_info = self._parse_json(
+            mobj.group('json_data'), video_id, transform_source=js_to_json)
+
+        if not isinstance(urls_info, list):
+            urls_info = [urls_info]
+
+        if mobj.group('type') == 'YouTube':
+            return self.playlist_result([
+                self.url_result(url_info['url']) for url_info in urls_info])
+
         formats = [{
             'url': url_info['url'],
             'width': int(url_info['width']),

From c0a656876cf3831deab90d0937e7cd81e95f6184 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 30 Aug 2015 01:11:56 +0800
Subject: [PATCH 1180/2145] [abc] Use int_or_none

---
 youtube_dl/extractor/abc.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 06acf50d8..f9a389f67 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     js_to_json,
+    int_or_none,
 )
 
 
@@ -59,10 +60,10 @@ class ABCIE(InfoExtractor):
 
         formats = [{
             'url': url_info['url'],
-            'width': int(url_info['width']),
-            'height': int(url_info['height']),
-            'tbr': int(url_info['bitrate']),
-            'filesize': int(url_info['filesize']),
+            'width': int_or_none(url_info.get('width')),
+            'height': int_or_none(url_info.get('height')),
+            'tbr': int_or_none(url_info.get('bitrate')),
+            'filesize': int_or_none(url_info.get('filesize')),
         } for url_info in urls_info]
         self._sort_formats(formats)
 

From c9afb51ceaeae7cbd74502e0451eb9163a1ab7fb Mon Sep 17 00:00:00 2001
From: Anssi Hannula <anssi.hannula@iki.fi>
Date: Mon, 15 Jun 2015 20:06:43 +0300
Subject: [PATCH 1181/2145] [youtube] Fix missing format details for 60fps DASH
 formats

60fps DASH formats do not appear in the DASH manifest, but the non-DASH
video info page does contain additional parameters for DASH formats that
we can parse.

Use those when they exist.

This fixes "bestvideo" not selecting 60fps formats over similar 30fps
formats just because the file size is unknown.
---
 youtube_dl/extractor/youtube.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ab6754154..3a2c7c562 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1243,7 +1243,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
-            url_map = {}
+            formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)
                 if 'itag' not in url_data or 'url' not in url_data:
@@ -1303,8 +1303,33 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     url += '&signature=' + signature
                 if 'ratebypass' not in url:
                     url += '&ratebypass=yes'
-                url_map[format_id] = url
-            formats = _map_to_format_list(url_map)
+
+                width = None
+                height = None
+                size_str = url_data.get('size', [''])[0]
+                if size_str.count('x') == 1:
+                    width, height = [int_or_none(x) for x in size_str.split('x')]
+
+                format_url = {
+                    'format_id': format_id,
+                    'url': url,
+                    'player_url': player_url,
+                    # As of this writing these are only defined for DASH formats:
+                    'filesize': int_or_none(url_data.get('clen', [None])[0]),
+                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], scale=1024),
+                    'width': width,
+                    'height': height,
+                    'fps': int_or_none(url_data.get('fps', [None])[0]),
+                }
+
+                # drop Nones so they do not overwrite the defaults from self._formats
+                format_url = dict((k, v) for k, v in format_url.items() if v is not None)
+
+                format_full = self._formats.get(format_id, {}).copy()
+                format_full.update(format_url)
+
+                formats.append(format_full)
+
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]
             url_map = self._extract_from_m3u8(manifest_url, video_id)

From aabc2be69316550cbec486bbde5695d1ae13ee9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Aug 2015 08:07:07 +0600
Subject: [PATCH 1182/2145] [youtube] Simplify and extract more metadata from
 url_encoded_fmt_stream_map (Closes #5993)

---
 youtube_dl/extractor/youtube.py | 53 ++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3a2c7c562..030ec70ca 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1304,32 +1304,49 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if 'ratebypass' not in url:
                     url += '&ratebypass=yes'
 
-                width = None
-                height = None
-                size_str = url_data.get('size', [''])[0]
-                if size_str.count('x') == 1:
-                    width, height = [int_or_none(x) for x in size_str.split('x')]
-
-                format_url = {
+                # Some itags are not included in DASH manifest thus corresponding formats will
+                # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
+                # Trying to extract metadata from url_encoded_fmt_stream_map entry.
+                mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
+                width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
+                dct = {
                     'format_id': format_id,
                     'url': url,
                     'player_url': player_url,
-                    # As of this writing these are only defined for DASH formats:
                     'filesize': int_or_none(url_data.get('clen', [None])[0]),
-                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], scale=1024),
+                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
                     'width': width,
                     'height': height,
                     'fps': int_or_none(url_data.get('fps', [None])[0]),
+                    'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
                 }
-
-                # drop Nones so they do not overwrite the defaults from self._formats
-                format_url = dict((k, v) for k, v in format_url.items() if v is not None)
-
-                format_full = self._formats.get(format_id, {}).copy()
-                format_full.update(format_url)
-
-                formats.append(format_full)
-
+                type_ = url_data.get('type', [None])[0]
+                if type_:
+                    type_split = type_.split(';')
+                    kind_ext = type_split[0].split('/')
+                    if len(kind_ext) == 2:
+                        kind, ext = kind_ext
+                        dct['ext'] = ext
+                        if kind in ('audio', 'video'):
+                            codecs = None
+                            for mobj in re.finditer(
+                                    r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
+                                if mobj.group('key') == 'codecs':
+                                    codecs = mobj.group('val')
+                                    break
+                            if codecs:
+                                codecs = codecs.split(',')
+                                if len(codecs) == 2:
+                                    acodec, vcodec = codecs[0], codecs[1]
+                                else:
+                                    acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
+                                dct.update({
+                                    'acodec': acodec,
+                                    'vcodec': vcodec,
+                                })
+                if format_id in self._formats:
+                    dct.update(self._formats[format_id])
+                formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]
             url_map = self._extract_from_m3u8(manifest_url, video_id)

From 8875b3d572ba37185fa632fa909bb6f9ecda8218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Aug 2015 19:03:21 +0600
Subject: [PATCH 1183/2145] [imgut:album] Add extractor (Closes #6711)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/imgur.py    | 27 ++++++++++++++++++++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 66422b005..39b05ce8f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -241,7 +241,10 @@ from .imdb import (
     ImdbIE,
     ImdbListIE
 )
-from .imgur import ImgurIE
+from .imgur import (
+    ImgurIE,
+    ImgurAlbumIE,
+)
 from .ina import InaIE
 from .indavideo import (
     IndavideoIE,
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index d692ea79a..70c8ca64e 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!gallery)(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -97,3 +97,28 @@ class ImgurIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'title': self._og_search_title(webpage),
         }
+
+
+class ImgurAlbumIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/gallery/(?P<id>[a-zA-Z0-9]+)'
+
+    _TEST = {
+        'url': 'http://imgur.com/gallery/Q95ko',
+        'info_dict': {
+            'id': 'Q95ko',
+        },
+        'playlist_count': 25,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        album_images = self._download_json(
+            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
+            album_id)['data']['images']
+
+        entries = [
+            self.url_result('http://imgur.com/%s' % image['hash'])
+            for image in album_images if image.get('hash')]
+
+        return self.playlist_result(entries, album_id)

From d3d89c325681b1f2d9dcc8d8dea3439aad570f1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 02:01:13 +0600
Subject: [PATCH 1184/2145] [YoutubeDL] Encode thumbnail filename (Closes
 #6714)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index cad6b026e..982e658ce 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2009,7 +2009,7 @@ class YoutubeDL(object):
                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
                 try:
                     uf = self.urlopen(t['url'])
-                    with open(thumb_filename, 'wb') as thumbf:
+                    with open(encodeFilename(thumb_filename), 'wb') as thumbf:
                         shutil.copyfileobj(uf, thumbf)
                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))

From db8f2bfd99fccd3ac4d96b49cf34056962a5652f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 02:20:29 +0600
Subject: [PATCH 1185/2145] [f4m] Add missing imports

---
 youtube_dl/downloader/f4m.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index f478fc03c..174180db5 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -13,6 +13,8 @@ from ..compat import (
     compat_urllib_error,
 )
 from ..utils import (
+    encodeFilename,
+    sanitize_open,
     struct_pack,
     struct_unpack,
     xpath_text,

From fcd9e423ec15ac51844786541e1f5cb464eb429a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 02:28:36 +0600
Subject: [PATCH 1186/2145] [hls] Properly encode fragment filename

---
 youtube_dl/downloader/hls.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 2b6c3370f..71aafdc73 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -12,6 +12,7 @@ from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
     encodeArgument,
     encodeFilename,
+    sanitize_open,
 )
 
 
@@ -89,13 +90,13 @@ class NativeHlsFD(FragmentFD):
             success = ctx['dl'].download(frag_filename, {'url': frag_url})
             if not success:
                 return False
-            with open(frag_filename, 'rb') as down:
-                ctx['dest_stream'].write(down.read())
-            frags_filenames.append(frag_filename)
+            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
+            ctx['dest_stream'].write(down.read())
+            frags_filenames.append(frag_sanitized)
 
         self._finish_frag_download(ctx)
 
         for frag_file in frags_filenames:
-            os.remove(frag_file)
+            os.remove(encodeFilename(frag_file))
 
         return True

From 615f155a3afd5c854fbf855c02f17ddb7f217b05 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 31 Aug 2015 12:46:28 +0800
Subject: [PATCH 1187/2145] [vlive] Fixes for review

---
 youtube_dl/extractor/vlive.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 6a403cc64..17ae3b964 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -39,7 +39,7 @@ class VLiveIE(InfoExtractor):
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<span class="name">([^<>]+)</span>', webpage, 'creator')
+            r'<span[^>]+class="name">([^<>]+)</span>', webpage, 'creator')
         
         url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
         msgpad = '%.0f' % (time() * 1000)
@@ -51,7 +51,7 @@ class VLiveIE(InfoExtractor):
         playinfo = self._download_json(url, video_id, 'Downloading video json')
 
         if playinfo.get('message', '') != 'success':
-            raise ExtractorError(playinfo['message'])
+            raise ExtractorError(playinfo.get('message', 'JSON request unsuccessful'))
 
         if not playinfo.get('result'):
             raise ExtractorError('No videos found.')

From 93462856e107237f85295357f978f2207d15723c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 21:05:23 +0600
Subject: [PATCH 1188/2145] [fc2] Relax _VALID_URL (Closes #3171)

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index e4f7195a8..1c6dc5bd9 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _NETRC_MACHINE = 'fc2'
     _TESTS = [{

From 52dfb7ffe2efdd223872bfe020ddaa25b89a94f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 21:05:37 +0600
Subject: [PATCH 1189/2145] [fc2] Add test for #3171

---
 youtube_dl/extractor/fc2.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 1c6dc5bd9..2443846c8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -37,6 +37,9 @@ class FC2IE(InfoExtractor):
             'password': '(snip)',
             'skip': 'requires actual password'
         }
+    }, {
+        'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
+        'only_matching': True,
     }]
 
     def _login(self):

From 39955b0451098c6a79fc7b777a0674ee093ae0ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 21:12:21 +0600
Subject: [PATCH 1190/2145] [fc2] Fix refer

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 2443846c8..5c1137e94 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -83,7 +83,7 @@ class FC2IE(InfoExtractor):
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        refer = url.replace('/content/', '/a/content/')
+        refer = url.replace('/content/', '/a/content/') if '/a/content/' not in url else url
 
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 

From d14f0c45fc8ab447d66caf7a3b9ce0888446eb7c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 1 Sep 2015 11:11:14 +0800
Subject: [PATCH 1191/2145] [vlive] Add info about  www. to m. redirects for
 mobile

---
 youtube_dl/extractor/vlive.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 17ae3b964..a456f8217 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -16,6 +16,7 @@ from ..compat import compat_urllib_parse
 
 class VLiveIE(InfoExtractor):
     IE_NAME = 'vlive'
+    # www.vlive.tv/video/ links redirect to m.vlive.tv/video/ for mobile devices
     _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://m.vlive.tv/video/1326',

From cdc8d0c37325e83da0c08ee4d97d81040e10be91 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 1 Sep 2015 16:34:05 +0800
Subject: [PATCH 1192/2145] [odnoklassniki] Fix extraction and extend
 _VALID_URL (closes #6726)

---
 youtube_dl/extractor/odnoklassniki.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 003d27de7..f552e6f7e 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
@@ -46,6 +46,9 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,
+    }, {
+        'url': 'http://www.ok.ru/video/20648036891',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -56,7 +59,7 @@ class OdnoklassnikiIE(InfoExtractor):
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-attributes="([^"]+)"', webpage, 'player')),
+                r'data-options="([^"]+)"', webpage, 'player')),
             video_id)
 
         flashvars = player['flashvars']

From fffccaaf414800e24a5ebc9e90c324aaa37cd3a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 20:41:52 +0600
Subject: [PATCH 1193/2145] [globo] Fix extraction and make more robust (Closes
 #6728)

---
 youtube_dl/extractor/globo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 8a95793ca..33d6432a6 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -13,6 +13,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
+    int_or_none,
 )
 
 
@@ -359,13 +360,8 @@ class GloboIE(InfoExtractor):
             self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
 
         title = video['title']
-        duration = float_or_none(video['duration'], 1000)
-        like_count = video['likes']
-        uploader = video['channel']
-        uploader_id = video['channel_id']
 
         formats = []
-
         for resource in video['resources']:
             resource_id = resource.get('_id')
             if not resource_id:
@@ -407,6 +403,11 @@ class GloboIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        duration = float_or_none(video.get('duration'), 1000)
+        like_count = int_or_none(video.get('likes'))
+        uploader = video.get('channel')
+        uploader_id = video.get('channel_id')
+
         return {
             'id': video_id,
             'title': title,

From aa3f98677d0ca7f8feae63cf2a5c62dbdda6b586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 1 Sep 2015 18:13:33 +0200
Subject: [PATCH 1194/2145] [dumpert] Support 'embed/' urls (fixes #6712)

---
 youtube_dl/extractor/dumpert.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 999fb5620..1f00386fe 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -9,8 +9,8 @@ from ..utils import qualities
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/mediabase/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
+    _TESTS = [{
         'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
         'info_dict': {
@@ -20,11 +20,15 @@ class DumpertIE(InfoExtractor):
             'description': 'Niet schrikken hoor',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.dumpert.nl/embed/6675421/dc440fe7/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        url = 'https://www.dumpert.nl/mediabase/' + video_id
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)

From 33eae08f0422fb824cea2a4dffc98a347e916169 Mon Sep 17 00:00:00 2001
From: PSJay <psjay.peng@gmail.com>
Date: Tue, 1 Sep 2015 00:19:58 +0800
Subject: [PATCH 1195/2145] Support video-password on Youku.

---
 youtube_dl/extractor/youku.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 78caeb8b3..220f84221 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -49,6 +49,17 @@ class YoukuIE(InfoExtractor):
         },
         'playlist_count': 13,
         'skip': 'Available in China only',
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
+        'note': 'Video protected with password',
+        'info_dict': {
+            'id': 'XNjA1NzA2Njgw',
+            'ext': 'flv',
+            'title': u'邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
+        },
+        'params': {
+            'videopassword': '100600',
+        },
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -185,10 +196,17 @@ class YoukuIE(InfoExtractor):
             raw_data = self._download_json(req, video_id, note=note)
             return raw_data['data'][0]
 
+        video_password = self._downloader.params.get('videopassword', None)
+
         # request basic data
-        data1 = retrieve_data(
-            'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
-            'Downloading JSON metadata 1')
+        if video_password:
+            data1 = retrieve_data(
+                'http://v.youku.com/player/getPlayList/VideoIDS/%s?password=%s' % (video_id, video_password),
+                'Downloading JSON metadata 1')
+        else:
+            data1 = retrieve_data(
+                'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
+                'Downloading JSON metadata 1')
         data2 = retrieve_data(
             'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
             'Downloading JSON metadata 2')

From cb3d2eb9e99e1a3f05a5009fc15afc0212a1af1e Mon Sep 17 00:00:00 2001
From: PSJay <psjay.peng@gmail.com>
Date: Tue, 1 Sep 2015 11:07:47 +0800
Subject: [PATCH 1196/2145] Fix coding styple in YouKuIE.

---
 youtube_dl/extractor/youku.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 220f84221..6219bc1bb 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -199,14 +199,13 @@ class YoukuIE(InfoExtractor):
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
+        basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
         if video_password:
-            data1 = retrieve_data(
-                'http://v.youku.com/player/getPlayList/VideoIDS/%s?password=%s' % (video_id, video_password),
-                'Downloading JSON metadata 1')
-        else:
-            data1 = retrieve_data(
-                'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
-                'Downloading JSON metadata 1')
+            basic_data_url = '%s?password=%s' % (basic_data_url, video_password)
+
+        data1 = retrieve_data(
+            basic_data_url,
+            'Downloading JSON metadata 1')
         data2 = retrieve_data(
             'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
             'Downloading JSON metadata 2')

From f859695b497717d40e164d0d6a3e29b3c813dad1 Mon Sep 17 00:00:00 2001
From: PSJay <psjay.peng@gmail.com>
Date: Tue, 1 Sep 2015 12:50:01 +0800
Subject: [PATCH 1197/2145] Update README

Add Youku in video password description.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 25844eb6d..b6f0677f7 100644
--- a/README.md
+++ b/README.md
@@ -207,7 +207,7 @@ which means you can modify it, redistribute it or use it however you like.
     -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
-    --video-password PASSWORD        Video password (vimeo, smotri)
+    --video-password PASSWORD        Video password (vimeo, smotri, Youku)
 
 ## Post-processing Options:
     -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)

From 5ddc127da6e2df440723e067b04bfb2ce86bb827 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:26:17 +0600
Subject: [PATCH 1198/2145] [youku] Simplify (Closes #6717)

---
 youtube_dl/extractor/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 6219bc1bb..3447e7a6f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -55,7 +55,7 @@ class YoukuIE(InfoExtractor):
         'info_dict': {
             'id': 'XNjA1NzA2Njgw',
             'ext': 'flv',
-            'title': u'邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
+            'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
         },
         'params': {
             'videopassword': '100600',
@@ -201,7 +201,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
         if video_password:
-            basic_data_url = '%s?password=%s' % (basic_data_url, video_password)
+            basic_data_url += '?password=%s' % video_password
 
         data1 = retrieve_data(
             basic_data_url,

From cd5d75427e0a739018c018af49152e0052b7a1fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:28:03 +0600
Subject: [PATCH 1199/2145] [youku] Fix videopassword test

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 3447e7a6f..2e81d9223 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -54,9 +54,9 @@ class YoukuIE(InfoExtractor):
         'note': 'Video protected with password',
         'info_dict': {
             'id': 'XNjA1NzA2Njgw',
-            'ext': 'flv',
             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
         },
+        'playlist_count': 19,
         'params': {
             'videopassword': '100600',
         },

From 804c343a4ff252c46fcc4a36f73a913c8def9b93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:30:28 +0600
Subject: [PATCH 1200/2145] [options] Clarify extractors supporting
 videopassword

---
 README.md             | 2 +-
 youtube_dl/options.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b6f0677f7..1d6c1151d 100644
--- a/README.md
+++ b/README.md
@@ -207,7 +207,7 @@ which means you can modify it, redistribute it or use it however you like.
     -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
-    --video-password PASSWORD        Video password (vimeo, smotri, Youku)
+    --video-password PASSWORD        Video password (vimeo, smotri, youku)
 
 ## Post-processing Options:
     -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9016e3498..8c4ff12bd 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -320,7 +320,7 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
-        help='Video password (vimeo, smotri)')
+        help='Video password (vimeo, smotri, youku)')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(

From 36c15522c121c0ffeb5208847a4f5461adc4f3b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:37:42 +0600
Subject: [PATCH 1201/2145] [francetv] Use subtitle when present (Closes #6715)

---
 youtube_dl/extractor/francetv.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 75723c00d..c053774b9 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -78,9 +78,14 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        title = info['titre']
+        subtitle = info.get('sous_titre')
+        if subtitle:
+            title += ' - %s' % subtitle
+
         return {
             'id': video_id,
-            'title': info['titre'],
+            'title': title,
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
             'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),

From 6917d2a2f0396f7fb6373e6160c6f25b8664aafb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:39:26 +0600
Subject: [PATCH 1202/2145] [francetv] Update tests

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index c053774b9..f70a6fea0 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -224,7 +224,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
             'info_dict': {
                 'id': '108961659',
                 'ext': 'flv',
-                'title': 'C à dire ?!',
+                'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
                 'description': 'md5:1a4aeab476eb657bf57c4ff122129f81',
                 'upload_date': '20140915',
                 'timestamp': 1410795000,

From 77306e8b970f462d202db5914d0382b176172f05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:42:43 +0600
Subject: [PATCH 1203/2145] [francetv] Update tests (2)

---
 youtube_dl/extractor/francetv.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index f70a6fea0..129984a5f 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -219,15 +219,15 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         },
         # france5
         {
-            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/92837968',
-            'md5': '78f0f4064f9074438e660785bbf2c5d9',
+            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/quels_sont_les_enjeux_de_cette_rentree_politique__31-08-2015_908948?onglet=tous&page=1',
+            'md5': 'f6c577df3806e26471b3d21631241fd0',
             'info_dict': {
-                'id': '108961659',
+                'id': '123327454',
                 'ext': 'flv',
                 'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
-                'description': 'md5:1a4aeab476eb657bf57c4ff122129f81',
-                'upload_date': '20140915',
-                'timestamp': 1410795000,
+                'description': 'md5:4a0d5cb5dce89d353522a84462bae5a4',
+                'upload_date': '20150831',
+                'timestamp': 1441035120,
             },
         },
         # franceo

From 080997b8083fed357061e301ab7f38484b24ded2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 1 Sep 2015 20:30:58 +0200
Subject: [PATCH 1204/2145] [rtve] Fix extraction (fixes #6723)

Adding the 'Referer' header to the png request is enough.
---
 youtube_dl/extractor/rtve.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 82cd98ac7..5b97d33ca 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,7 +6,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import compat_urllib_request, compat_urlparse
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -102,7 +102,9 @@ class RTVEALaCartaIE(InfoExtractor):
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
-        png = self._download_webpage(png_url, video_id, 'Downloading url information')
+        png_request = compat_urllib_request.Request(png_url)
+        png_request.add_header('Referer', url)
+        png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
             auth_url = video_url.replace(

From e5e78797e6e9873b93e550c32f454bbd5444a4ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 02:16:04 +0600
Subject: [PATCH 1205/2145] [utils] Strict HTTP responses (Closes #6727)

---
 youtube_dl/utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e265c7574..b7a423166 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -587,6 +587,11 @@ class ContentTooShortError(Exception):
 
 
 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
+    # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
+    # expected HTTP responses to meet HTTP/1.0 or later (see also
+    # https://github.com/rg3/youtube-dl/issues/6727)
+    if sys.version_info < (3, 0):
+        kwargs['strict'] = True
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')
     if source_address is not None:

From ebf4ca39baa0ac2f38cbeebc1c76f02524002c91 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Sep 2015 01:24:59 +0100
Subject: [PATCH 1206/2145] link to developer instructions and correct
 common.py path

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 1d6c1151d..722f11679 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the developer instructions below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
+Alternatively, refer to the [developer instructions](#developer-instructions) below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
@@ -552,7 +552,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From 8f9d522f6252f874d41045b2458256ca8cf1a312 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 21:09:31 +0600
Subject: [PATCH 1207/2145] [README.md] Remove superfluous word

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 722f11679..24bfe38a2 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from

From 198492bbf0e497079158bdf9ff94df3c91d6f3b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 21:36:22 +0600
Subject: [PATCH 1208/2145] [ok] Extract correct data-options (Closes #6726)

---
 youtube_dl/extractor/odnoklassniki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index f552e6f7e..97900a4c3 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -59,7 +59,7 @@ class OdnoklassnikiIE(InfoExtractor):
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-options="([^"]+)"', webpage, 'player')),
+                r'data-options="([^"]+%s[^"]+)"' % video_id, webpage, 'player')),
             video_id)
 
         flashvars = player['flashvars']

From 1e804244d062c7620bcc9498cba1c7d3d8bd8095 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 21:38:56 +0600
Subject: [PATCH 1209/2145] [ok] Improve player regex

---
 youtube_dl/extractor/odnoklassniki.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 97900a4c3..4564fd0be 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -59,7 +59,8 @@ class OdnoklassnikiIE(InfoExtractor):
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-options="([^"]+%s[^"]+)"' % video_id, webpage, 'player')),
+                r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
+                webpage, 'player', group='player')),
             video_id)
 
         flashvars = player['flashvars']

From 88720ed09b7921c967432fc3f4aa7a994e8028a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 22:08:50 +0600
Subject: [PATCH 1210/2145] [ok] Add support for youtube embeds

---
 youtube_dl/extractor/odnoklassniki.py | 46 ++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 4564fd0be..a077b676f 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -43,6 +43,21 @@ class OdnoklassnikiIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
         },
+    }, {
+        # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
+        'url': 'http://ok.ru/video/64211978996595-1',
+        'md5': '5d7475d428845cd2e13bae6f1a992278',
+        'info_dict': {
+            'id': '64211978996595-1',
+            'ext': 'mp4',
+            'title': 'Космическая среда от 26 августа 2015',
+            'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
+            'duration': 440,
+            'upload_date': '20150826',
+            'uploader_id': '750099571',
+            'uploader': 'Алина П',
+            'age_limit': 0,
+        },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,
@@ -93,16 +108,7 @@ class OdnoklassnikiIE(InfoExtractor):
 
         like_count = int_or_none(metadata.get('likeCount'))
 
-        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
-
-        formats = [{
-            'url': f['url'],
-            'ext': 'mp4',
-            'format_id': f['name'],
-            'quality': quality(f['name']),
-        } for f in metadata['videos']]
-
-        return {
+        info = {
             'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
@@ -112,5 +118,23 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': uploader_id,
             'like_count': like_count,
             'age_limit': age_limit,
-            'formats': formats,
         }
+
+        if metadata.get('provider') == 'USER_YOUTUBE':
+            info.update({
+                '_type': 'url_transparent',
+                'url': movie['contentId'],
+            })
+            return info
+
+        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
+
+        formats = [{
+            'url': f['url'],
+            'ext': 'mp4',
+            'format_id': f['name'],
+            'quality': quality(f['name']),
+        } for f in metadata['videos']]
+
+        info['formats'] = formats
+        return info

From e8dcfa3d698fd328760e40f6a76f99c45dfcdc85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 22:09:33 +0600
Subject: [PATCH 1211/2145] [ok] Sort formats

---
 youtube_dl/extractor/odnoklassniki.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index a077b676f..66520c2c5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -135,6 +135,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'format_id': f['name'],
             'quality': quality(f['name']),
         } for f in metadata['videos']]
+        self._sort_formats(formats)
 
         info['formats'] = formats
         return info

From bd6742137f532d2a3c09c8b89543618ae982abf5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 3 Sep 2015 12:34:20 +0200
Subject: [PATCH 1212/2145] release 2015.09.03

---
 CONTRIBUTING.md        | 2 +-
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 42333c450..f8ab29631 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -125,7 +125,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 328a819b3..04b9959ac 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -220,6 +220,7 @@
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
  - **Imgur**
+ - **ImgurAlbum**
  - **Ina**
  - **Indavideo**
  - **IndavideoEmbed**
@@ -303,6 +304,7 @@
  - **MPORA**
  - **MSNBC**
  - **MTV**
+ - **mtv.de**
  - **mtviggy.com**
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a07bc9233..6bc689b75 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.28'
+__version__ = '2015.09.03'

From 725d1c58aa25da8640eef8d62b9451cbd6762169 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 3 Sep 2015 13:15:02 +0200
Subject: [PATCH 1213/2145] [crunchyroll] Extract width and height (closes
 #6749)

---
 youtube_dl/extractor/crunchyroll.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c2162aa68..ce123482e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -20,9 +20,11 @@ from ..utils import (
     ExtractorError,
     bytes_to_intlist,
     intlist_to_bytes,
+    int_or_none,
     remove_end,
     unified_strdate,
     urlencode_postdata,
+    xpath_text,
 )
 from ..aes import (
     aes_cbc_decrypt,
@@ -286,6 +288,13 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             stream_info = streamdata.find('./{default}preload/stream_info')
             video_url = stream_info.find('./host').text
             video_play_path = stream_info.find('./file').text
+            metadata = stream_info.find('./metadata')
+            format_info = {
+                'format': video_format,
+                'format_id': video_format,
+                'height': int_or_none(xpath_text(metadata, './height')),
+                'width': int_or_none(xpath_text(metadata, './width')),
+            }
 
             if '.fplive.net/' in video_url:
                 video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
@@ -294,19 +303,18 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                     netloc='v.lvlt.crcdn.net',
                     path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
                 if self._is_valid_url(direct_video_url, video_id, video_format):
-                    formats.append({
+                    format_info.update({
                         'url': direct_video_url,
-                        'format_id': video_format,
                     })
+                    formats.append(format_info)
                     continue
 
-            formats.append({
+            format_info.update({
                 'url': video_url,
                 'play_path': video_play_path,
                 'ext': 'flv',
-                'format': video_format,
-                'format_id': video_format,
             })
+            formats.append(format_info)
 
         subtitles = self.extract_subtitles(video_id, webpage)
 

From e7a8c3032d8610bc429b6318678387433728b2a9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Sep 2015 22:25:33 +0100
Subject: [PATCH 1214/2145] [downloader/external] Respect
 --no-check-certificate for curl and aria2c and --proxy for curl

---
 youtube_dl/downloader/external.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 6c310346c..072ac3a46 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _option(self, command_option, param):
         param = self.params.get(param)
-        if param is None:
+        if param is None or param is False:
             return []
         if isinstance(param, bool):
             return [command_option]
@@ -80,6 +80,8 @@ class CurlFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
+        cmd += self._option('--proxy', 'proxy')
+        cmd += self._option('--insecure', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -121,6 +123,7 @@ class Aria2cFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
+        cmd += self._option('--check-certificate=false', 'nocheckcertificate')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 26c61e080908b20f3c6707789bb42933bbbb42a0 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Fri, 4 Sep 2015 12:59:13 +0200
Subject: [PATCH 1215/2145] Added ec extension to nowvideo extractor

---
 youtube_dl/extractor/nowvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index dec09cdfe..17baa9679 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,7 +7,7 @@ class NowVideoIE(NovaMovIE):
     IE_NAME = 'nowvideo'
     IE_DESC = 'NowVideo'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co|li)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|ec|sx|eu|at|ag|co|li)'}
 
     _HOST = 'www.nowvideo.ch'
 

From a41fb80ce1e2fec48d7a4bc15d169061e76ca672 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Sep 2015 23:56:45 +0600
Subject: [PATCH 1216/2145] [utils] Add xpath_element and xpath_attr

---
 youtube_dl/utils.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b7a423166..de5069c7b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -176,7 +176,7 @@ def xpath_with_ns(path, ns_map):
     return '/'.join(replaced)
 
 
-def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
+def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     if sys.version_info < (2, 7):  # Crazy 2.6
         xpath = xpath.encode('ascii')
 
@@ -189,7 +189,24 @@ def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
             raise ExtractorError('Could not find XML element %s' % name)
         else:
             return None
-    return n.text
+    return n
+
+
+def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
+    return xpath_element(node, xpath, name, fatal=fatal, default=default).text
+
+
+def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
+    n = find_xpath_attr(node, xpath, key)
+    if n is None:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
+            name = '%s[@%s]' % (xpath, key) if name is None else name
+            raise ExtractorError('Could not find XML attribute %s' % name)
+        else:
+            return None
+    return n.attrib[key]
 
 
 def get_element_by_id(id, html):

From 5d2354f1779182f260b4ef44ef6e92a3b5944183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Sep 2015 23:57:27 +0600
Subject: [PATCH 1217/2145] [utils] Relax attribute key assert

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index de5069c7b..86b757103 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -141,7 +141,7 @@ def write_json_file(obj, fn):
 if sys.version_info >= (2, 7):
     def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
-        assert re.match(r'^[a-zA-Z-]+$', key)
+        assert re.match(r'^[a-zA-Z_-]+$', key)
         if val:
             assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))

From 22889ab175d990d50050603fb983b1db7d1615d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 00:01:40 +0600
Subject: [PATCH 1218/2145] [ruutu] Fix extraction (Closes #6759)

---
 youtube_dl/extractor/ruutu.py | 41 +++++++++--------------------------
 1 file changed, 10 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 4e22628d0..8ca6e56bb 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -6,19 +6,19 @@ from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     determine_ext,
     int_or_none,
+    xpath_attr,
     xpath_text,
 )
 
 
 class RuutuIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ruutu\.fi/ohjelmat/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?ruutu\.fi/video/(?P<id>\d+)'
     _TESTS = [
         {
-            'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+            'url': 'http://www.ruutu.fi/video/2058907',
             'md5': 'ab2093f39be1ca8581963451b3c0234f',
             'info_dict': {
                 'id': '2058907',
-                'display_id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
                 'ext': 'mp4',
                 'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
                 'description': 'md5:cfc6ccf0e57a814360df464a91ff67d6',
@@ -32,10 +32,9 @@ class RuutuIE(InfoExtractor):
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
                 'id': '2057306',
-                'display_id': 'superpesis-katso-koko-kausi-ruudussa',
                 'ext': 'mp4',
                 'title': 'Superpesis: katso koko kausi Ruudussa',
-                'description': 'md5:44c44a99fdbe5b380ab74ebd75f0af77',
+                'description': 'md5:da2736052fef3b2bd5e0005e63c25eac',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 40,
                 'age_limit': 0,
@@ -44,29 +43,10 @@ class RuutuIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._search_regex(
-            r'data-media-id="(\d+)"', webpage, 'media id')
-
-        video_xml_url = None
-
-        media_data = self._search_regex(
-            r'jQuery\.extend\([^,]+,\s*(.+?)\);', webpage,
-            'media data', default=None)
-        if media_data:
-            media_json = self._parse_json(media_data, display_id, fatal=False)
-            if media_json:
-                xml_url = media_json.get('ruutuplayer', {}).get('xmlUrl')
-                if xml_url:
-                    video_xml_url = xml_url.replace('{ID}', video_id)
-
-        if not video_xml_url:
-            video_xml_url = 'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id
-
-        video_xml = self._download_xml(video_xml_url, video_id)
+        video_xml = self._download_xml(
+            'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id, video_id)
 
         formats = []
         processed_urls = []
@@ -109,10 +89,9 @@ class RuutuIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'display_id': display_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'title': xpath_attr(video_xml, './/Behavior/Program', 'program_name', 'title', fatal=True),
+            'description': xpath_attr(video_xml, './/Behavior/Program', 'description', 'description'),
+            'thumbnail': xpath_attr(video_xml, './/Behavior/Startpicture', 'href', 'thumbnail'),
             'duration': int_or_none(xpath_text(video_xml, './/Runtime', 'duration')),
             'age_limit': int_or_none(xpath_text(video_xml, './/AgeLimit', 'age limit')),
             'formats': formats,

From 8e636da4990643084de17c03c6a0639e9aa4085e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 00:34:49 +0600
Subject: [PATCH 1219/2145] [utils] Improve xpath_text

---
 youtube_dl/utils.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86b757103..cc792be6a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -181,7 +181,7 @@ def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
-    if n is None or n.text is None:
+    if n is None:
         if default is not NO_DEFAULT:
             return default
         elif fatal:
@@ -193,7 +193,18 @@ def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
 
 
 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
-    return xpath_element(node, xpath, name, fatal=fatal, default=default).text
+    n = xpath_element(node, xpath, name, fatal=fatal, default=default)
+    if n is None or n == default:
+        return n
+    if n.text is None:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
+            name = xpath if name is None else name
+            raise ExtractorError('Could not find XML element\'s text %s' % name)
+        else:
+            return None
+    return n.text
 
 
 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):

From 87f70ab39d0f65e4462403c0afcad0969171c356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 00:36:16 +0600
Subject: [PATCH 1220/2145] [test_utils] Add more tests for xpath

---
 test/test_utils.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a759b2da9..15a59692f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -57,7 +57,9 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    xpath_element,
     xpath_text,
+    xpath_attr,
     render_table,
     match_str,
     parse_dfxp_time_expr,
@@ -264,6 +266,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find('media:song/media:author').text, 'The Author')
         self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
 
+    def test_xpath_element(self):
+        doc = xml.etree.ElementTree.Element('root')
+        div = xml.etree.ElementTree.SubElement(doc, 'div')
+        p = xml.etree.ElementTree.SubElement(div, 'p')
+        p.text = 'Foo'
+        self.assertEqual(xpath_element(doc, 'div/p'), p)
+        self.assertEqual(xpath_element(doc, 'div/bar', default='default'), 'default')
+        self.assertTrue(xpath_element(doc, 'div/bar') is None)
+        self.assertRaises(ExtractorError, xpath_element, doc, 'div/bar', fatal=True)
+
     def test_xpath_text(self):
         testxml = '''<root>
             <div>
@@ -272,9 +284,25 @@ class TestUtil(unittest.TestCase):
         </root>'''
         doc = xml.etree.ElementTree.fromstring(testxml)
         self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
+        self.assertEqual(xpath_text(doc, 'div/bar', default='default'), 'default')
         self.assertTrue(xpath_text(doc, 'div/bar') is None)
         self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True)
 
+    def test_xpath_attr(self):
+        testxml = '''<root>
+            <div>
+                <p x="a">Foo</p>
+            </div>
+        </root>'''
+        doc = xml.etree.ElementTree.fromstring(testxml)
+        self.assertEqual(xpath_attr(doc, 'div/p', 'x'), 'a')
+        self.assertEqual(xpath_attr(doc, 'div/bar', 'x'), None)
+        self.assertEqual(xpath_attr(doc, 'div/p', 'y'), None)
+        self.assertEqual(xpath_attr(doc, 'div/bar', 'x', default='default'), 'default')
+        self.assertEqual(xpath_attr(doc, 'div/p', 'y', default='default'), 'default')
+        self.assertRaises(ExtractorError, xpath_attr, doc, 'div/bar', 'x', fatal=True)
+        self.assertRaises(ExtractorError, xpath_attr, doc, 'div/p', 'y', fatal=True)
+
     def test_smuggle_url(self):
         data = {"ö": "ö", "abc": [3]}
         url = 'https://foo.bar/baz?x=y#a'

From 266b0ad6762eaa48156b021094126062cb9f44d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 20:07:36 +0100
Subject: [PATCH 1221/2145] [downloader/external] add _bool_option to pass
 value to bool option

---
 youtube_dl/downloader/external.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 072ac3a46..22dfd2833 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -53,6 +53,14 @@ class ExternalFD(FileDownloader):
             return [command_option]
         return [command_option, param]
 
+    def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
+        param = self.params.get(param)
+        if not isinstance(param, bool):
+            return []
+        if separator:
+            return [command_option + separator + (true_value if param else false_value)]
+        return [command_option, true_value if param else false_value]
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -123,7 +131,7 @@ class Aria2cFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
-        cmd += self._option('--check-certificate=false', 'nocheckcertificate')
+        cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
         cmd += ['--', info_dict['url']]
         return cmd
 

From c482b3c69aac9069a8c1acb1f2b60d6578331461 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Sep 2015 21:11:29 +0200
Subject: [PATCH 1222/2145] [ruutu] Update test url

---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 8ca6e56bb..c67ad25ce 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -28,7 +28,7 @@ class RuutuIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
+            'url': 'http://www.ruutu.fi/video/2057306',
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
                 'id': '2057306',

From f30c2e8e9867373815aaaa2285f23524a67da730 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 20:57:19 +0100
Subject: [PATCH 1223/2145] [downloader/external] add _argless_option for
 option without arguments

---
 youtube_dl/downloader/external.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 22dfd2833..cae8f6f0f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -49,8 +49,6 @@ class ExternalFD(FileDownloader):
         param = self.params.get(param)
         if param is None or param is False:
             return []
-        if isinstance(param, bool):
-            return [command_option]
         return [command_option, param]
 
     def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
@@ -61,6 +59,10 @@ class ExternalFD(FileDownloader):
             return [command_option + separator + (true_value if param else false_value)]
         return [command_option, true_value if param else false_value]
 
+    def _argless_option(self, command_option, param, expected_value=True):
+        param = self.params.get(param)
+        return [command_option] if param == expected_value else []
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -89,7 +91,7 @@ class CurlFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._option('--insecure', 'nocheckcertificate')
+        cmd += self._argless_option('--insecure', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -112,7 +114,7 @@ class WgetFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._option('--no-check-certificate', 'nocheckcertificate')
+        cmd += self._argless_option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From dc534b674f9dfbe869d23f6a64ea48f39bde6655 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 21:12:13 +0100
Subject: [PATCH 1224/2145] [downloader/external] change _argless_option
 function to _valueless_option

---
 youtube_dl/downloader/external.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index cae8f6f0f..056215f8c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _option(self, command_option, param):
         param = self.params.get(param)
-        if param is None or param is False:
+        if param is None:
             return []
         return [command_option, param]
 
@@ -59,7 +59,7 @@ class ExternalFD(FileDownloader):
             return [command_option + separator + (true_value if param else false_value)]
         return [command_option, true_value if param else false_value]
 
-    def _argless_option(self, command_option, param, expected_value=True):
+    def _valueless_option(self, command_option, param, expected_value=True):
         param = self.params.get(param)
         return [command_option] if param == expected_value else []
 
@@ -91,7 +91,7 @@ class CurlFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._argless_option('--insecure', 'nocheckcertificate')
+        cmd += self._valueless_option('--insecure', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -114,7 +114,7 @@ class WgetFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._argless_option('--no-check-certificate', 'nocheckcertificate')
+        cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From 66e289bab466079558b7acf5cea1057ae35c9bfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:05:11 +0600
Subject: [PATCH 1225/2145] [utils] Generalize cli option converters

---
 youtube_dl/utils.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cc792be6a..79381b380 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1951,6 +1951,32 @@ def dfxp2srt(dfxp_data):
     return ''.join(out)
 
 
+def cli_option(params, command_option, param):
+    param = params.get(param)
+    return [command_option, param] if param is not None else []
+
+
+def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
+    param = params.get(param)
+    assert isinstance(param, bool)
+    if separator:
+        return [command_option + separator + (true_value if param else false_value)]
+    return [command_option, true_value if param else false_value]
+
+
+def cli_valueless_option(params, command_option, param, expected_value=True):
+    param = params.get(param)
+    return [command_option] if param == expected_value else []
+
+
+def cli_configuration_args(params, param, default=[]):
+    ex_args = params.get(param)
+    if ex_args is None:
+        return default
+    assert isinstance(ex_args, list)
+    return ex_args
+
+
 class ISO639Utils(object):
     # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
     _lang_map = {

From 1195a38f460a6bf895ada6405284da103dc467de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:06:28 +0600
Subject: [PATCH 1226/2145] [downloader/external] Use generalized cli option
 converters

---
 youtube_dl/downloader/external.py | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 056215f8c..2bc011266 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -5,6 +5,10 @@ import subprocess
 
 from .common import FileDownloader
 from ..utils import (
+    cli_option,
+    cli_valueless_option,
+    cli_bool_option,
+    cli_configuration_args,
     encodeFilename,
     encodeArgument,
 )
@@ -46,29 +50,16 @@ class ExternalFD(FileDownloader):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
     def _option(self, command_option, param):
-        param = self.params.get(param)
-        if param is None:
-            return []
-        return [command_option, param]
+        return cli_option(self.params, command_option, param)
 
     def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
-        param = self.params.get(param)
-        if not isinstance(param, bool):
-            return []
-        if separator:
-            return [command_option + separator + (true_value if param else false_value)]
-        return [command_option, true_value if param else false_value]
+        return cli_bool_option(self.params, command_option, param, true_value, false_value, separator)
 
     def _valueless_option(self, command_option, param, expected_value=True):
-        param = self.params.get(param)
-        return [command_option] if param == expected_value else []
+        return cli_valueless_option(self.params, command_option, param, expected_value)
 
     def _configuration_args(self, default=[]):
-        ex_args = self.params.get('external_downloader_args')
-        if ex_args is None:
-            return default
-        assert isinstance(ex_args, list)
-        return ex_args
+        return cli_configuration_args(self.params, 'external_downloader_args', default)
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """

From 6270239a6d75be02a13aec10bb7f229370ffea6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:06:53 +0600
Subject: [PATCH 1227/2145] [postprocessor/common] Use generalized cli option
 converters

---
 youtube_dl/postprocessor/common.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 4191d040b..150ef9173 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -4,6 +4,7 @@ import os
 
 from ..utils import (
     PostProcessingError,
+    cli_configuration_args,
     encodeFilename,
 )
 
@@ -61,11 +62,7 @@ class PostProcessor(object):
             self._downloader.report_warning(errnote)
 
     def _configuration_args(self, default=[]):
-        pp_args = self._downloader.params.get('postprocessor_args')
-        if pp_args is None:
-            return default
-        assert isinstance(pp_args, list)
-        return pp_args
+        return cli_configuration_args(self.params, 'postprocessor_args', default)
 
 
 class AudioConversionError(PostProcessingError):

From f71264490cbd1bb9d953aafdf0a37aae956c24d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:07:19 +0600
Subject: [PATCH 1228/2145] [test_utils] Add tests for cli option converters

---
 test/test_utils.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 15a59692f..a5f164c49 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -64,6 +64,9 @@ from youtube_dl.utils import (
     match_str,
     parse_dfxp_time_expr,
     dfxp2srt,
+    cli_option,
+    cli_valueless_option,
+    cli_bool_option,
 )
 
 
@@ -674,6 +677,51 @@ The first line
 '''
         self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
 
+    def test_cli_option(self):
+        self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
+        self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
+        self.assertEqual(cli_option({}, '--proxy', 'proxy'), [])
+
+    def test_cli_valueless_option(self):
+        self.assertEqual(cli_valueless_option(
+            {'downloader': 'external'}, '--external-downloader', 'downloader', 'external'), ['--external-downloader'])
+        self.assertEqual(cli_valueless_option(
+            {'downloader': 'internal'}, '--external-downloader', 'downloader', 'external'), [])
+        self.assertEqual(cli_valueless_option(
+            {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate'), ['--no-check-certificate'])
+        self.assertEqual(cli_valueless_option(
+            {'nocheckcertificate': False}, '--no-check-certificate', 'nocheckcertificate'), [])
+        self.assertEqual(cli_valueless_option(
+            {'checkcertificate': True}, '--no-check-certificate', 'checkcertificate', False), [])
+        self.assertEqual(cli_valueless_option(
+            {'checkcertificate': False}, '--no-check-certificate', 'checkcertificate', False), ['--no-check-certificate'])
+
+    def test_cli_bool_option(self):
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate'),
+            ['--no-check-certificate', 'true'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate', separator='='),
+            ['--no-check-certificate=true'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--check-certificate', 'nocheckcertificate', 'false', 'true'),
+            ['--check-certificate', 'false'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
+            ['--check-certificate=false'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true'),
+            ['--check-certificate', 'true'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
+            ['--check-certificate=true'])
+
 
 if __name__ == '__main__':
     unittest.main()

From 08354db47bba1138798c164c6bab8024600fd057 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Sep 2015 15:25:19 +0800
Subject: [PATCH 1229/2145] [vlive] PEP8

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index a456f8217..86c1cb5ef 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -41,7 +41,7 @@ class VLiveIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
             r'<span[^>]+class="name">([^<>]+)</span>', webpage, 'creator')
-        
+
         url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
         msgpad = '%.0f' % (time() * 1000)
         md = b64encode(

From 9dd73ef4a49597e73388e7e97e32eae5d0f0f027 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Sep 2015 15:38:05 +0800
Subject: [PATCH 1230/2145] [dcn] PEP8

---
 youtube_dl/extractor/dcn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 82261e25c..6f2fea5ff 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -44,8 +44,8 @@ class DCNIE(InfoExtractor):
         title = video.get('title_en') or video['title_ar']
 
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
-            + compat_urllib_parse.urlencode({
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
+            compat_urllib_parse.urlencode({
                 'id': video['id'],
                 'user_id': video['user_id'],
                 'signature': video['signature'],

From 3026164b1693ef57d0712cd6eeed6ae7a79b0916 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 5 Sep 2015 12:11:36 +0200
Subject: [PATCH 1231/2145] [postprocessor/common] Use
 'self._downloader.params' instead of 'self.params'

'self.params' is not defined
---
 youtube_dl/postprocessor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 150ef9173..599dd1df2 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -62,7 +62,7 @@ class PostProcessor(object):
             self._downloader.report_warning(errnote)
 
     def _configuration_args(self, default=[]):
-        return cli_configuration_args(self.params, 'postprocessor_args', default)
+        return cli_configuration_args(self._downloader.params, 'postprocessor_args', default)
 
 
 class AudioConversionError(PostProcessingError):

From 4abf617b9c662851889de4a3ceb5aa63040dc21e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 19:17:30 +0600
Subject: [PATCH 1232/2145] [YoutubeDL] Ensure bool params always present in
 downloader

---
 youtube_dl/YoutubeDL.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 982e658ce..0afda5ecb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -284,7 +284,11 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
-        self.params = params
+        self.params = {
+            # Default parameters
+            'nocheckcertificate': False,
+        }
+        self.params.update(params)
         self.cache = Cache(self)
 
         if params.get('bidi_workaround', False):

From c23c3d7d7d3ca9368b6fca70b696fe64ca55ba07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 16:38:05 +0100
Subject: [PATCH 1233/2145] [nowness] fix video extraction and add support
 serie and playlist extraction (fixes #6720)

---
 youtube_dl/extractor/__init__.py |   6 +-
 youtube_dl/extractor/nowness.py  | 109 +++++++++++++++++++++++--------
 2 files changed, 88 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..7408f68d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -403,7 +403,11 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
 from .novamov import NovaMovIE
-from .nowness import NownessIE
+from .nowness import (
+    NownessIE,
+    NownessPlaylistIE,
+    NownessSerieIE,
+)
 from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
 from .npo import (
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 6b2f3f55a..1acb9b92a 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,19 +1,42 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .brightcove import BrightcoveIE
 from .common import InfoExtractor
 from ..utils import ExtractorError
+from ..compat import compat_urllib_request
 
 
-class NownessIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])'
+class NownessBaseIE(InfoExtractor):
+    def extract_url_result(self, post):
+        if post['type'] == 'video':
+            for media in post['media']:
+                if media['type'] == 'video':
+                    video_id = media['content']
+                    source = media['source']
+                    if source == 'brightcove':
+                        player_code = self._download_webpage(
+                            'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
+                            note='Downloading player JavaScript',
+                            errnote='Player download failed')
+                        bc_url = BrightcoveIE._extract_brightcove_url(player_code)
+                        if bc_url is None:
+                            raise ExtractorError('Could not find player definition')
+                        return self.url_result(bc_url, 'Brightcove')
+                    elif source == 'vimeo':
+                        return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
+                    elif source == 'youtube':
+                        return self.url_result(video_id, 'Youtube')
+                    elif source == 'cinematique':
+                        return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
+
+class NownessIE(NownessBaseIE):
+    IE_NAME = 'nowness'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/])/(?P<id>[0-9a-z-]+)'
     _TESTS = [
         {
-            'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation',
+            'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
             'md5': '068bc0202558c2e391924cb8cc470676',
             'info_dict': {
                 'id': '2520295746001',
@@ -25,7 +48,7 @@ class NownessIE(InfoExtractor):
             }
         },
         {
-            'url': 'http://cn.nowness.com/day/2014/8/7/4069/kasper-bj-rke-ft-jaakko-eino-kalevi--tnr',
+            'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
             'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
             'info_dict': {
                 'id': '3716354522001',
@@ -39,26 +62,60 @@ class NownessIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('slug')
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        player_url = self._search_regex(
-            r'"([^"]+/content/issue-[0-9.]+.js)"', webpage, 'player URL')
-        real_id = self._search_regex(
-            r'\sdata-videoId="([0-9]+)"', webpage, 'internal video ID')
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request('http://api.nowness.com/api/post/getBySlug/%s' % display_id, headers={
+            'X-Nowness-Language': lang,
+        })
+        post = self._download_json(request, display_id)
+        return self.extract_url_result(post)
 
-        player_code = self._download_webpage(
-            player_url, video_id,
-            note='Downloading player JavaScript',
-            errnote='Player download failed')
-        player_code = player_code.replace("'+d+'", real_id)
 
-        bc_url = BrightcoveIE._extract_brightcove_url(player_code)
-        if bc_url is None:
-            raise ExtractorError('Could not find player definition')
-        return {
-            '_type': 'url',
-            'url': bc_url,
-            'ie_key': 'Brightcove',
-        }
+class NownessPlaylistIE(NownessBaseIE):
+    IE_NAME = 'nowness:playlist'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)/[0-9a-z-]+'
+    _TEST = {
+        'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
+        'info_dict':
+        {
+            'id': '3286',
+        },
+        'playlist_mincount': 8,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request('http://api.nowness.com/api/post?PlaylistId=%s' % playlist_id, headers={
+            'X-Nowness-Language': lang,
+        })
+        playlist = self._download_json(request, playlist_id)
+        entries = [self.extract_url_result(item) for item in playlist['items']]
+        return self.playlist_result(entries, playlist_id)
+
+
+class NownessSerieIE(NownessBaseIE):
+    IE_NAME = 'nowness:serie'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[0-9a-z-]+)'
+    _TEST = {
+        'url': 'https://www.nowness.com/series/60-seconds',
+        'info_dict':
+        {
+            'id': '60',
+        },
+        'playlist_mincount': 4,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request('https://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
+            'X-Nowness-Language': lang,
+        })
+        serie = self._download_json(request, display_id)
+        serie_id = str(serie['id'])
+        entries = [self.extract_url_result(post) for post in serie['posts']]
+        return self.playlist_result(entries, serie_id)

From 51f579b6358ab56ff9d03ef72e5bfeb345632b4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:40:44 +0600
Subject: [PATCH 1234/2145] [compat] Add compat shlex.split

---
 youtube_dl/compat.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index ace5bd716..9b77316f9 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -5,6 +5,7 @@ import getpass
 import optparse
 import os
 import re
+import shlex
 import shutil
 import socket
 import subprocess
@@ -227,6 +228,17 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
+if sys.version_info > (2, 7, 2):
+    compat_shlex_split = shlex.split
+else:
+    # Working around shlex issue with unicode strings on some python 2
+    # versions (see http://bugs.python.org/issue1548891)
+    def compat_shlex_split(s, comments=False, posix=True):
+        if isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return shlex.split(s, comments, posix)
+
+
 def compat_ord(c):
     if type(c) is int:
         return c
@@ -459,6 +471,7 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
+    'compat_shlex_split',
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',

From ee087c79adc0cd94458d6fa0191acb4592bb0e45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:41:34 +0600
Subject: [PATCH 1235/2145] [test_compat] Add test for compat_shlex_split

---
 test/test_compat.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index c3ba8ad2e..d41a25e8e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
+    compat_shlex_split,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
 )
@@ -67,5 +68,8 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
         self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
 
+    def test_compat_shlex(self):
+        self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
+
 if __name__ == '__main__':
     unittest.main()

From 8870358b1b1a13c936277081b668f94c5abf95e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:42:22 +0600
Subject: [PATCH 1236/2145] [__init__] Use compat_shlex_split

---
 youtube_dl/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 55b22c889..5e2ed4d4b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -9,7 +9,6 @@ import codecs
 import io
 import os
 import random
-import shlex
 import sys
 
 
@@ -20,6 +19,7 @@ from .compat import (
     compat_expanduser,
     compat_getpass,
     compat_print,
+    compat_shlex_split,
     workaround_optparse_bug9161,
 )
 from .utils import (
@@ -262,10 +262,10 @@ def _real_main(argv=None):
             parser.error('setting filesize xattr requested but python-xattr is not available')
     external_downloader_args = None
     if opts.external_downloader_args:
-        external_downloader_args = shlex.split(opts.external_downloader_args)
+        external_downloader_args = compat_shlex_split(opts.external_downloader_args)
     postprocessor_args = None
     if opts.postprocessor_args:
-        postprocessor_args = shlex.split(opts.postprocessor_args)
+        postprocessor_args = compat_shlex_split(opts.postprocessor_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))

From cab792abe521d36e1cf296fa8578a37a499a78fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:42:41 +0600
Subject: [PATCH 1237/2145] [options] Use compat_shlex_split

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 8c4ff12bd..5eccc0a70 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import os.path
 import optparse
-import shlex
 import sys
 
 from .downloader.external import list_external_downloaders
@@ -11,6 +10,7 @@ from .compat import (
     compat_get_terminal_size,
     compat_getenv,
     compat_kwargs,
+    compat_shlex_split,
 )
 from .utils import (
     preferredencoding,
@@ -28,7 +28,7 @@ def parseOpts(overrideArguments=None):
         try:
             res = []
             for l in optionf:
-                res += shlex.split(l, comments=True)
+                res += compat_shlex_split(l, comments=True)
         finally:
             optionf.close()
         return res

From 3513d4143657c1bdc7da72f048a63d4b1ee794d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:45:52 +0600
Subject: [PATCH 1238/2145] [test_compat] Fix typo

---
 test/test_compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index d41a25e8e..4ee0dc99d 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -68,7 +68,7 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
         self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
 
-    def test_compat_shlex(self):
+    def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
 if __name__ == '__main__':

From 12439dd5ec46ccac1ef13db7280473ecef44a096 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Thu, 3 Sep 2015 07:13:05 +0200
Subject: [PATCH 1239/2145] [generic] Capture <audio> tags in addition to
 <video> tags

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 953ec32c3..ec748ed9f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1797,7 +1797,7 @@ class GenericIE(InfoExtractor):
                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
         if not found:
             # HTML5 video
-            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
+            found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(

From 75b399f45557468e602c3b0126a9b5ce4736218f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 16:53:17 +0100
Subject: [PATCH 1240/2145] [nowness] fix _VALID_URL regex

---
 youtube_dl/extractor/nowness.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 1acb9b92a..401d3b646 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -33,7 +33,7 @@ class NownessBaseIE(InfoExtractor):
 
 class NownessIE(NownessBaseIE):
     IE_NAME = 'nowness'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/])/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/]+)/(?P<id>[0-9a-z-]+)'
     _TESTS = [
         {
             'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
@@ -112,7 +112,7 @@ class NownessSerieIE(NownessBaseIE):
         display_id = self._match_id(url)
 
         lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('https://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
+        request = compat_urllib_request.Request('http://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
             'X-Nowness-Language': lang,
         })
         serie = self._download_json(request, display_id)

From 8df5ae15d1f010bd96a8c51087718c5f41d3d90f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 22:21:06 +0600
Subject: [PATCH 1241/2145] [compat] Fix python version check for
 compat_shlex_split

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9b77316f9..e32bef279 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -228,7 +228,7 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
-if sys.version_info > (2, 7, 2):
+if sys.version_info >= (2, 7, 3):
     compat_shlex_split = shlex.split
 else:
     # Working around shlex issue with unicode strings on some python 2

From f33f32f1591087743454cbaa6a6689299f1da1d4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 20:32:31 +0100
Subject: [PATCH 1242/2145] [nowness] add api abstration function adn extend
 _VALID_URL regex

---
 youtube_dl/extractor/nowness.py | 40 +++++++++++++--------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 401d3b646..04d614c9c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -30,10 +30,20 @@ class NownessBaseIE(InfoExtractor):
                     elif source == 'cinematique':
                         return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
+    def api_request(self, url, request_url):
+        id = self._match_id(url)
+
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request(request_url % id, headers={
+            'X-Nowness-Language': lang,
+        })
+        json_data = self._download_json(request, id)
+        return id, json_data
+
 
 class NownessIE(NownessBaseIE):
     IE_NAME = 'nowness'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/]+)/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P<id>[^/]+?)(?:$|[?#])'
     _TESTS = [
         {
             'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
@@ -62,19 +72,13 @@ class NownessIE(NownessBaseIE):
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/post/getBySlug/%s' % display_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        post = self._download_json(request, display_id)
+        display_id, post = self.api_request(url, 'http://api.nowness.com/api/post/getBySlug/%s')
         return self.extract_url_result(post)
 
 
 class NownessPlaylistIE(NownessBaseIE):
     IE_NAME = 'nowness:playlist'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)/[0-9a-z-]+'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
         'info_dict':
@@ -85,20 +89,14 @@ class NownessPlaylistIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/post?PlaylistId=%s' % playlist_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        playlist = self._download_json(request, playlist_id)
+        playlist_id, playlist = self.api_request(url, 'http://api.nowness.com/api/post?PlaylistId=%s')
         entries = [self.extract_url_result(item) for item in playlist['items']]
         return self.playlist_result(entries, playlist_id)
 
 
 class NownessSerieIE(NownessBaseIE):
     IE_NAME = 'nowness:serie'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[^/]+?)(?:$|[?#])'
     _TEST = {
         'url': 'https://www.nowness.com/series/60-seconds',
         'info_dict':
@@ -109,13 +107,7 @@ class NownessSerieIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        serie = self._download_json(request, display_id)
+        display_id, serie = self.api_request(url, 'http://api.nowness.com/api/series/getBySlug/%s')
         serie_id = str(serie['id'])
         entries = [self.extract_url_result(post) for post in serie['posts']]
         return self.playlist_result(entries, serie_id)

From f95c5e1218aacc5cc4c01b19f5ee9f7baf9269d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 20:51:27 +0100
Subject: [PATCH 1243/2145] [nowness] change id variable name

---
 youtube_dl/extractor/nowness.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 04d614c9c..a8ee4fe68 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -31,14 +31,14 @@ class NownessBaseIE(InfoExtractor):
                         return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
     def api_request(self, url, request_url):
-        id = self._match_id(url)
+        display_id = self._match_id(url)
 
         lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request(request_url % id, headers={
+        request = compat_urllib_request.Request(request_url % display_id, headers={
             'X-Nowness-Language': lang,
         })
-        json_data = self._download_json(request, id)
-        return id, json_data
+        json_data = self._download_json(request, display_id)
+        return display_id, json_data
 
 
 class NownessIE(NownessBaseIE):

From 673bf566fc1aa1351a40c7378893aff6cb0fc91b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 21:03:13 +0100
Subject: [PATCH 1244/2145] [nowness] more api abstraction

---
 youtube_dl/extractor/nowness.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index a8ee4fe68..3371145c1 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -30,11 +30,11 @@ class NownessBaseIE(InfoExtractor):
                     elif source == 'cinematique':
                         return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
-    def api_request(self, url, request_url):
+    def api_request(self, url, request_path):
         display_id = self._match_id(url)
 
         lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request(request_url % display_id, headers={
+        request = compat_urllib_request.Request('http://api.nowness.com/api/' + request_path % display_id, headers={
             'X-Nowness-Language': lang,
         })
         json_data = self._download_json(request, display_id)
@@ -72,7 +72,7 @@ class NownessIE(NownessBaseIE):
     ]
 
     def _real_extract(self, url):
-        display_id, post = self.api_request(url, 'http://api.nowness.com/api/post/getBySlug/%s')
+        display_id, post = self.api_request(url, 'post/getBySlug/%s')
         return self.extract_url_result(post)
 
 
@@ -89,7 +89,7 @@ class NownessPlaylistIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        playlist_id, playlist = self.api_request(url, 'http://api.nowness.com/api/post?PlaylistId=%s')
+        playlist_id, playlist = self.api_request(url, 'post?PlaylistId=%s')
         entries = [self.extract_url_result(item) for item in playlist['items']]
         return self.playlist_result(entries, playlist_id)
 
@@ -107,7 +107,7 @@ class NownessSerieIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        display_id, serie = self.api_request(url, 'http://api.nowness.com/api/series/getBySlug/%s')
+        display_id, serie = self.api_request(url, 'series/getBySlug/%s')
         serie_id = str(serie['id'])
         entries = [self.extract_url_result(post) for post in serie['posts']]
         return self.playlist_result(entries, serie_id)

From f43c1631582267fba591d150a18bee764bb37708 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 21:18:21 +0100
Subject: [PATCH 1245/2145] [nowness] pass cinematique

---
 youtube_dl/extractor/nowness.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 3371145c1..9a0e46917 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -28,7 +28,9 @@ class NownessBaseIE(InfoExtractor):
                     elif source == 'youtube':
                         return self.url_result(video_id, 'Youtube')
                     elif source == 'cinematique':
-                        return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
+                        # youtube-dl currently doesn't support cinematique
+                        # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
+                        pass
 
     def api_request(self, url, request_path):
         display_id = self._match_id(url)

From eb387896e90d3c0b9043169769783796c564f4a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 02:24:14 +0600
Subject: [PATCH 1246/2145] [adultswim] Fix extraction on python 2.6

---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 39335b827..4327c2f61 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -156,7 +156,7 @@ class AdultSwimIE(InfoExtractor):
                 xpath_text(idoc, './/trt', 'segment duration').strip())
 
             formats = []
-            file_els = idoc.findall('.//files/file')
+            file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')
 
             for file_el in file_els:
                 bitrate = file_el.attrib.get('bitrate')

From a6420bf50cc60aa9c8e37bfb2acb10eca361a4f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 06:20:48 +0600
Subject: [PATCH 1247/2145] [utils] Add cookie processor for cookie correction
 (Closes #6769)

---
 youtube_dl/utils.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 79381b380..9bc292a39 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -782,6 +782,32 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             req, **kwargs)
 
 
+class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
+    def __init__(self, cookiejar=None):
+        compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
+
+    def http_response(self, request, response):
+        # Python 2 will choke on next HTTP request in row if there are non-ASCII
+        # characters in Set-Cookie HTTP header of last response (see
+        # https://github.com/rg3/youtube-dl/issues/6769).
+        # In order to at least prevent crashing we will percent encode Set-Cookie
+        # header before HTTPCookieProcessor starts processing it.
+        if sys.version_info < (3, 0) and response.headers:
+            for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
+                set_cookie = response.headers.get(set_cookie_header)
+                if set_cookie:
+                    set_cookie_escaped = '; '.join([
+                        escape_rfc3986(cookie_attr.strip())
+                        for cookie_attr in set_cookie.decode('iso-8859-1').split(';')]).encode('iso-8859-1')
+                    if set_cookie != set_cookie_escaped:
+                        del response.headers[set_cookie_header]
+                        response.headers[set_cookie_header] = set_cookie_escaped
+        return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
+
+    https_request = compat_urllib_request.HTTPCookieProcessor.http_request
+    https_response = http_response
+
+
 def parse_iso8601(date_str, delimiter='T', timezone=None):
     """ Return a UNIX timestamp from the given date """
 

From 6a3f4c3f829eaff87caacf162ae4fc60045bd059 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 06:21:33 +0600
Subject: [PATCH 1248/2145] [YoutubeDL] Use custom cookie processor

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0afda5ecb..d65253882 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -69,6 +69,7 @@ from .utils import (
     version_tuple,
     write_json_file,
     write_string,
+    YoutubeDLCookieProcessor,
     YoutubeDLHandler,
     prepend_extension,
     replace_extension,
@@ -1943,8 +1944,7 @@ class YoutubeDL(object):
             if os.access(opts_cookiefile, os.R_OK):
                 self.cookiejar.load()
 
-        cookie_processor = compat_urllib_request.HTTPCookieProcessor(
-            self.cookiejar)
+        cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
         if opts_proxy is not None:
             if opts_proxy == '':
                 proxies = {}

From ad7291727499590a9449b1e3ba3d7a33b733ddd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 06:23:44 +0600
Subject: [PATCH 1249/2145] [utils] Add issue URL in comment for #6457

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9bc292a39..7cf91ac19 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -748,7 +748,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
-        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
+        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
+        # https://github.com/rg3/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:
             location = resp.headers.get('Location')
             if location:

From be0e5dbd8356e92f8032d4cf8be2e668f1b30539 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:20:47 +0600
Subject: [PATCH 1250/2145] [extractor/common] Extract submit inputs

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 39cef9c5b..5eeeda08d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -733,7 +733,7 @@ class InfoExtractor(object):
     def _hidden_inputs(html):
         hidden_inputs = {}
         for input in re.findall(r'<input([^>]+)>', html):
-            if not re.search(r'type=(["\'])hidden\1', input):
+            if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
                 continue
             name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
             if not name:

From 1639282434c77f4de005fb0a6b81a49425e64d08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:22:20 +0600
Subject: [PATCH 1251/2145] [utils] Add encode_dict

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7cf91ac19..5529d5291 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1638,6 +1638,10 @@ def urlencode_postdata(*args, **kargs):
     return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
 
 
+def encode_dict(d, encoding='utf-8'):
+    return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
+
+
 try:
     etree_iter = xml.etree.ElementTree.Element.iter
 except AttributeError:  # Python <=2.6

From e213c98df179270a33b67ef64fa53a73f93ab4c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:23:53 +0600
Subject: [PATCH 1252/2145] [gorillavid] Build correct URL

---
 youtube_dl/extractor/gorillavid.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index f006f0cb1..6226b90c8 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -73,7 +73,8 @@ class GorillaVidIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        webpage = self._download_webpage('http://%s/%s' % (mobj.group('host'), video_id), video_id)
+        url = 'http://%s/%s' % (mobj.group('host'), video_id)
+        webpage = self._download_webpage(url, video_id)
 
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)

From e94cb5ae7e61627b3772c57c85195d0ce5fcf5bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:24:47 +0600
Subject: [PATCH 1253/2145] [gorillavid] Add support for filehoot.com

---
 youtube_dl/extractor/gorillavid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 6226b90c8..1024a5c22 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -15,10 +15,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 

From 4abe2144994dce59b823d2bc7ea3ddc90d68b2c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:30:02 +0600
Subject: [PATCH 1254/2145] [gorillavid] Encode fields

---
 youtube_dl/extractor/gorillavid.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 1024a5c22..1a42de2d5 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    encode_dict,
     int_or_none,
 )
 
@@ -88,7 +89,7 @@ class GorillaVidIE(InfoExtractor):
             if countdown:
                 self._sleep(countdown, video_id)
 
-            post = compat_urllib_parse.urlencode(fields)
+            post = compat_urllib_parse.urlencode(encode_dict(fields))
 
             req = compat_urllib_request.Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')

From b3e64671cc7a33386c2e484da8eddcc694d85302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:30:16 +0600
Subject: [PATCH 1255/2145] [gorillavid] Improve title

---
 youtube_dl/extractor/gorillavid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 1a42de2d5..afdf4e4c4 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -97,7 +97,7 @@ class GorillaVidIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = self._search_regex(
-            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'>Watch (.+) '],
+            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'<td nowrap>([^<]+)</td>', r'>Watch (.+) '],
             webpage, 'title', default=None) or self._og_search_title(webpage)
         video_url = self._search_regex(
             r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')

From c7c0996d8c2136dc863e45169a19def3269562a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:32:06 +0600
Subject: [PATCH 1256/2145] [gorillavid] Add test for filehoot.com (Closes
 #6771)

---
 youtube_dl/extractor/gorillavid.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index afdf4e4c4..d23e3eac1 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -68,6 +68,14 @@ class GorillaVidIE(InfoExtractor):
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,
+    }, {
+        'url': 'http://filehoot.com/3ivfabn7573c.html',
+        'info_dict': {
+            'id': '3ivfabn7573c',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
+            'thumbnail': 're:http://.*\.jpg',
+        }
     }]
 
     def _real_extract(self, url):

From 515fc8776fa1fff6ab3b1e2445ddccf5a22c520b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:42:15 +0600
Subject: [PATCH 1257/2145] [youtube] Use encode_dict

---
 youtube_dl/extractor/youtube.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 030ec70ca..231c86c37 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,6 +26,7 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
+    encode_dict,
     ExtractorError,
     float_or_none,
     get_element_by_attribute,
@@ -111,10 +112,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'hl': 'en_US',
         }
 
-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
+        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
 
         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
@@ -147,8 +145,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'TrustDevice': 'on',
             })
 
-            tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
-            tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
+            tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
 
             tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(

From cd019668dc06b3fec1de9168a7414f3bcc3fe52b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:42:45 +0600
Subject: [PATCH 1258/2145] [niconico] Use encode_dict

---
 youtube_dl/extractor/niconico.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 0f8aa5ada..bda1cff05 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -12,6 +12,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     int_or_none,
     parse_duration,
@@ -100,10 +101,7 @@ class NiconicoIE(InfoExtractor):
             'mail': username,
             'password': password,
         }
-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
         request = compat_urllib_request.Request(
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(

From cf83f532ae8a21dd58e7506d1496f611c372f861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:43:05 +0600
Subject: [PATCH 1259/2145] [niconico] Use encode_dict

---
 youtube_dl/extractor/fc2.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 5c1137e94..a406945e8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
 )
 
@@ -55,10 +56,7 @@ class FC2IE(InfoExtractor):
             'Submit': ' Login ',
         }
 
-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
         request = compat_urllib_request.Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 

From 266e466ee48f510226de9c7eb99f8727c99175bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:53:11 +0600
Subject: [PATCH 1260/2145] [utils] Simplify cookie processor

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5529d5291..9c547729e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -797,9 +797,7 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
             for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
                 set_cookie = response.headers.get(set_cookie_header)
                 if set_cookie:
-                    set_cookie_escaped = '; '.join([
-                        escape_rfc3986(cookie_attr.strip())
-                        for cookie_attr in set_cookie.decode('iso-8859-1').split(';')]).encode('iso-8859-1')
+                    set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
                     if set_cookie != set_cookie_escaped:
                         del response.headers[set_cookie_header]
                         response.headers[set_cookie_header] = set_cookie_escaped

From 12bbd32ad0993d512ef798d5a56891d82fb29d12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 08:00:16 +0600
Subject: [PATCH 1261/2145] [kuwo:singer] Skip test

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 1077846f2..fa233377d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -202,6 +202,7 @@ class KuwoSingerIE(InfoExtractor):
             'title': 'Ali',
         },
         'playlist_mincount': 95,
+        'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
     }]
 
     def _real_extract(self, url):

From e28034c5ac4e08414ba63decae8bf7d956fa6d93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 08:16:39 +0600
Subject: [PATCH 1262/2145] [utils] Comment cookie processing until result from
 travis and some more testing

---
 youtube_dl/utils.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c547729e..206dd56bc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -793,14 +793,14 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
         # https://github.com/rg3/youtube-dl/issues/6769).
         # In order to at least prevent crashing we will percent encode Set-Cookie
         # header before HTTPCookieProcessor starts processing it.
-        if sys.version_info < (3, 0) and response.headers:
-            for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
-                set_cookie = response.headers.get(set_cookie_header)
-                if set_cookie:
-                    set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
-                    if set_cookie != set_cookie_escaped:
-                        del response.headers[set_cookie_header]
-                        response.headers[set_cookie_header] = set_cookie_escaped
+        # if sys.version_info < (3, 0) and response.headers:
+        #     for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
+        #         set_cookie = response.headers.get(set_cookie_header)
+        #         if set_cookie:
+        #             set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
+        #             if set_cookie != set_cookie_escaped:
+        #                 del response.headers[set_cookie_header]
+        #                 response.headers[set_cookie_header] = set_cookie_escaped
         return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
 
     https_request = compat_urllib_request.HTTPCookieProcessor.http_request

From 7465222a9cbcc67f932ffc3468086317518becd7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 6 Sep 2015 04:45:13 +0100
Subject: [PATCH 1263/2145] [airmozilla] fix test info

---
 youtube_dl/extractor/airmozilla.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
index 611ad1e9d..f8e70f4e5 100644
--- a/youtube_dl/extractor/airmozilla.py
+++ b/youtube_dl/extractor/airmozilla.py
@@ -20,14 +20,14 @@ class AirMozillaIE(InfoExtractor):
             'id': '6x4q2w',
             'ext': 'mp4',
             'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
-            'thumbnail': 're:https://\w+\.cloudfront\.net/6x4q2w/poster\.jpg\?t=\d+',
+            'thumbnail': 're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
             'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
             'timestamp': 1422487800,
             'upload_date': '20150128',
             'location': 'SFO Commons',
             'duration': 3780,
             'view_count': int,
-            'categories': ['Main'],
+            'categories': ['Main', 'Privacy'],
         }
     }
 

From a2f42a3baf03f1b8ac7180337fe77ab1fd6fdd2c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 6 Sep 2015 04:52:27 +0100
Subject: [PATCH 1264/2145] [academicearth] fix test info

---
 youtube_dl/extractor/academicearth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py
index 47313fba8..34095501c 100644
--- a/youtube_dl/extractor/academicearth.py
+++ b/youtube_dl/extractor/academicearth.py
@@ -15,7 +15,7 @@ class AcademicEarthCourseIE(InfoExtractor):
             'title': 'Laws of Nature',
             'description': 'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.',
         },
-        'playlist_count': 4,
+        'playlist_count': 3,
     }
 
     def _real_extract(self, url):

From bca553caac52551e9222eb5bb19f3145e0c09768 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 10:21:15 +0600
Subject: [PATCH 1265/2145] [eagleplatform] Fix extraction

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 688dfc2f7..a1ee51568 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -79,7 +79,7 @@ class EaglePlatformIE(InfoExtractor):
             age_limit = 0 if age_restriction == 'allow_all' else 18
 
         m3u8_data = self._download_json(
-            media['sources']['secure_m3u8']['auto'],
+            self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:'),
             video_id, 'Downloading m3u8 JSON')
 
         formats = self._extract_m3u8_formats(

From 9af461de351f6c9594e693969ec6b6a7938479bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 10:28:40 +0600
Subject: [PATCH 1266/2145] [techtvmit] Fix extraction

---
 youtube_dl/extractor/mit.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index d7ab6a9ae..6cef41159 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -33,8 +33,8 @@ class TechTVMITIE(InfoExtractor):
             'http://techtv.mit.edu/videos/%s' % video_id, video_id)
         clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page)
 
-        base_url = self._search_regex(
-            r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url')
+        base_url = self._proto_relative_url(self._search_regex(
+            r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url'), 'http:')
         formats_json = self._search_regex(
             r'bitrates: (\[.+?\])', raw_page, 'video formats')
         formats_mit = json.loads(formats_json)

From 9dbdb65abe11c8b7c56354b78431879d2ddee52a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 10:32:42 +0600
Subject: [PATCH 1267/2145] [techtvmit] Update test

---
 youtube_dl/extractor/mit.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 6cef41159..f088ab9e2 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -18,12 +18,12 @@ class TechTVMITIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
-        'md5': '1f8cb3e170d41fd74add04d3c9330e5f',
+        'md5': '00a3a27ee20d44bcaa0933ccec4a2cf7',
         'info_dict': {
             'id': '25418',
             'ext': 'mp4',
-            'title': 'MIT DNA Learning Center Set',
-            'description': 'md5:82313335e8a8a3f243351ba55bc1b474',
+            'title': 'MIT DNA and Protein Sets',
+            'description': 'md5:46f5c69ce434f0a97e7c628cc142802d',
         },
     }
 

From 97243fe3957769cc91f6a4c936dcccfa356f82db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Sep 2015 04:52:26 +0600
Subject: [PATCH 1268/2145] [ceskatelevize] Extract playlists (Closes #6772)

---
 youtube_dl/extractor/ceskatelevize.py | 156 ++++++++++++++++----------
 1 file changed, 96 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index dda583680..e857e66f4 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -17,55 +17,81 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P<id>[^?#]+)'
-
-    _TESTS = [
-        {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
-            'info_dict': {
-                'id': '214411058091220',
-                'ext': 'mp4',
-                'title': 'Hyde Park Civilizace',
-                'description': 'Věda a současná civilizace. Interaktivní pořad - prostor pro vaše otázky a komentáře',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 3350,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
+    _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _TESTS = [{
+        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
+        'info_dict': {
+            'id': '61924494876951776',
+            'ext': 'mp4',
+            'title': 'Hyde Park Civilizace',
+            'description': 'md5:fe93f6eda372d150759d11644ebbfb4a',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3350,
         },
-        {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
-            'info_dict': {
-                'id': '14716',
-                'ext': 'mp4',
-                'title': 'První republika: Zpěvačka z Dupárny Bobina',
-                'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 88.4,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-    ]
+    }, {
+        'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
+        'info_dict': {
+            'id': '61924494876844374',
+            'ext': 'mp4',
+            'title': 'První republika: Zpěvačka z Dupárny Bobina',
+            'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 88.4,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Queer: Bogotart',
+            'description': 'Alternativní průvodce současným queer světem',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494876844842',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Varování 18+)',
+                'duration': 10.2,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Queer)',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         url = url.replace('/porady/', '/ivysilani/').replace('/video/', '')
 
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        playlist_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, playlist_id)
 
         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        typ = self._html_search_regex(r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
-        episode_id = self._html_search_regex(r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
+        typ = self._html_search_regex(
+            r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
+        episode_id = self._html_search_regex(
+            r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
 
         data = {
             'playlist[0][type]': typ,
@@ -83,7 +109,7 @@ class CeskaTelevizeIE(InfoExtractor):
         req.add_header('X-Requested-With', 'XMLHttpRequest')
         req.add_header('Referer', url)
 
-        playlistpage = self._download_json(req, video_id)
+        playlistpage = self._download_json(req, playlist_id)
 
         playlist_url = playlistpage['url']
         if playlist_url == 'error_region':
@@ -92,33 +118,43 @@ class CeskaTelevizeIE(InfoExtractor):
         req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
-        playlist = self._download_json(req, video_id)
+        playlist_title = self._og_search_title(webpage)
+        playlist_description = self._og_search_description(webpage)
 
-        item = playlist['playlist'][0]
-        formats = []
-        for format_id, stream_url in item['streamUrls'].items():
-            formats.extend(self._extract_m3u8_formats(stream_url, video_id, 'mp4'))
-        self._sort_formats(formats)
+        playlist = self._download_json(req, playlist_id)['playlist']
+        playlist_len = len(playlist)
 
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        duration = float_or_none(item.get('duration'))
-        thumbnail = item.get('previewImageUrl')
+        entries = []
+        for item in playlist:
+            formats = []
+            for format_id, stream_url in item['streamUrls'].items():
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url, playlist_id, 'mp4', entry_protocol='m3u8_native'))
+            self._sort_formats(formats)
 
-        subtitles = {}
-        subs = item.get('subtitles')
-        if subs:
-            subtitles = self.extract_subtitles(episode_id, subs)
+            item_id = item.get('id') or item['assetId']
+            title = item['title']
 
-        return {
-            'id': episode_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+            duration = float_or_none(item.get('duration'))
+            thumbnail = item.get('previewImageUrl')
+
+            subtitles = {}
+            if item.get('type') == 'VOD':
+                subs = item.get('subtitles')
+                if subs:
+                    subtitles = self.extract_subtitles(episode_id, subs)
+
+            entries.append({
+                'id': item_id,
+                'title': playlist_title if playlist_len == 1 else '%s (%s)' % (playlist_title, title),
+                'description': playlist_description if playlist_len == 1 else None,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            })
+
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
     def _get_subtitles(self, episode_id, subs):
         original_subtitles = self._download_webpage(

From daeb0f04cdd70a72ab9919340c7340d734fc5068 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 08:09:34 +0100
Subject: [PATCH 1269/2145] [yahoo] change test md5 checksum

---
 youtube_dl/extractor/yahoo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index f9afbdbab..b8579b573 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -101,7 +101,7 @@ class YahooIE(InfoExtractor):
             }
         }, {
             'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
-            'md5': '67010fdf3a08d290e060a4dd96baa07b',
+            'md5': '88e209b417f173d86186bef6e4d1f160',
             'info_dict': {
                 'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
                 'ext': 'mp4',

From cf33a47df0806a2df349b8e2629d7ffa2c62cbdc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 08:22:38 +0100
Subject: [PATCH 1270/2145] [ajazzera] skip test

---
 youtube_dl/extractor/aljazeera.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 612708e25..184a14a4f 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -16,6 +16,7 @@ class AlJazeeraIE(InfoExtractor):
             'uploader': 'Al Jazeera English',
         },
         'add_ie': ['Brightcove'],
+        'skip': 'Not accessible from Travis CI server',
     }
 
     def _real_extract(self, url):

From c67a055d16075fb59d365f4f259af6c68d4db76b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 07:09:55 +0100
Subject: [PATCH 1271/2145] [test/test_write_annotations] fix test filename

Closes #6781
---
 test/test_write_annotations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index 780636c77..84b8f39e0 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -33,7 +33,7 @@ params = get_params({
 
 
 TEST_ID = 'gr51aVj-mLg'
-ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'
+ANNOTATIONS_FILE = TEST_ID + '.annotations.xml'
 EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']
 
 

From 3b18f539a7e122004f4010981b044738985cf871 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 11:18:21 +0100
Subject: [PATCH 1272/2145] [ndr] fix info extraction

---
 youtube_dl/extractor/ndr.py | 104 +++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 79a13958b..77b7e5eb8 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -1,68 +1,66 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    qualities,
-    parse_duration,
 )
 
 
+preferences = {'xl': 4, 'l': 3, 'm': 2, 's': 1, 'xs': 0,}
+
+
 class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        page = self._download_webpage(url, video_id, 'Downloading page')
-
-        title = self._og_search_title(page).strip()
-        description = self._og_search_description(page)
-        if description:
-            description = description.strip()
-
-        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', default=None))
-        if not duration:
-            duration = parse_duration(self._html_search_regex(
-                r'(<span class="min">\d+</span>:<span class="sec">\d+</span>)',
-                page, 'duration', default=None))
+        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, 'Downloading page')
 
         formats = []
-
-        mp3_url = re.search(r'''\{src:'(?P<audio>[^']+)', type:"audio/mp3"},''', page)
-        if mp3_url:
-            formats.append({
-                'url': mp3_url.group('audio'),
-                'format_id': 'mp3',
-            })
-
-        thumbnail = None
-
-        video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.(lo|hi|hq)\.mp4', type:"video/mp4"},''', page)
-        if video_url:
-            thumbnails = re.findall(r'''\d+: \{src: "([^"]+)"(?: \|\| '[^']+')?, quality: '([^']+)'}''', page)
-            if thumbnails:
-                quality_key = qualities(['xs', 's', 'm', 'l', 'xl'])
-                largest = max(thumbnails, key=lambda thumb: quality_key(thumb[1]))
-                thumbnail = 'http://www.ndr.de' + largest[0]
-
-            for format_id in 'lo', 'hi', 'hq':
-                formats.append({
-                    'url': '%s.%s.mp4' % (video_url.group('video'), format_id),
-                    'format_id': format_id,
-                })
-
-        if not formats:
+        objetType = json_data.get('config').get('objectType')
+        if objetType == 'video':
+            for key, f in json_data.get('playlist').items():
+                if key != 'config':
+                    src = f['src']
+                    if '.f4m' in src:
+                        formats.extend(self._extract_f4m_formats(src, video_id))
+                    elif '.m3u8' in src:
+                        formats.extend(self._extract_m3u8_formats(src, video_id))
+                    else:
+                        quality = f.get('quality')
+                        formats.append({
+                            'url': src,
+                            'format_id': quality,
+                            'preference': preferences.get(quality),
+                        })
+        elif objetType == 'audio':
+            for key, f in json_data.get('playlist').items():
+                if key != 'config':
+                    formats.append({
+                        'url': f['src'],
+                        'format_id': 'mp3',
+                        
+                    })
+        else:
             raise ExtractorError('No media links available for %s' % video_id)
 
+        self._sort_formats(formats)
+
+        config = json_data.get('playlist').get('config')
+
+        title = config['title']
+        duration = int_or_none(config.get('duration'))
+        thumbnails = [{
+            'id': thumbnail.get('quality'),
+            'url': thumbnail.get('src'),
+            'preference': preferences.get(thumbnail.get('quality'))
+        } for thumbnail in config.get('poster').values()]
+
         return {
             'id': video_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'duration': duration,
             'formats': formats,
         }
@@ -71,7 +69,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://www\.ndr\.de/.+?,(?P<id>\w+)\.html'
 
     _TESTS = [
         {
@@ -79,10 +77,9 @@ class NDRIE(NDRBaseIE):
             'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
             'note': 'Video file',
             'info_dict': {
-                'id': '25866',
+                'id': 'nordmagazin25866',
                 'ext': 'mp4',
                 'title': 'Kartoffeltage in der Lewitz',
-                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
                 'duration': 166,
             },
             'skip': '404 Not found',
@@ -91,22 +88,20 @@ class NDRIE(NDRBaseIE):
             'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
             'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
             'info_dict': {
-                'id': '988',
+                'id': 'hafengeburtstag988',
                 'ext': 'mp4',
                 'title': 'Party, Pötte und Parade',
-                'description': 'Hunderttausende feiern zwischen Speicherstadt und St. Pauli den 826. Hafengeburtstag. Die NDR Sondersendung zeigt die schönsten und spektakulärsten Bilder vom Auftakt.',
                 'duration': 3498,
             },
         },
         {
-            'url': 'http://www.ndr.de/info/audio51535.html',
+            'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
             'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
             'note': 'Audio file',
             'info_dict': {
-                'id': '51535',
+                'id': 'audio51535',
                 'ext': 'mp3',
                 'title': 'La Valette entgeht der Hinrichtung',
-                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
                 'duration': 884,
             }
         }
@@ -115,7 +110,7 @@ class NDRIE(NDRBaseIE):
 
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/.+?(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/.+?,(?P<id>\w+)\.html'
 
     _TEST = {
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
@@ -124,7 +119,6 @@ class NJoyIE(NDRBaseIE):
             'id': '2480',
             'ext': 'mp4',
             'title': 'Benaissa beim NDR Comedy Contest',
-            'description': 'Von seinem sehr "behaarten" Leben lässt sich Benaissa trotz aller Schwierigkeiten nicht unterkriegen.',
             'duration': 654,
         }
     }

From 8801255d7deb6b3cbbbae75d14d855b39b309090 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 07:47:55 +0100
Subject: [PATCH 1273/2145] [youtube] fix show extraction (fixes #4841)

Closes #6782)
---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 231c86c37..6d7b90016 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1836,7 +1836,7 @@ class YoutubeShowIE(InfoExtractor):
     IE_NAME = 'youtube:show'
     _TESTS = [{
         'url': 'http://www.youtube.com/show/airdisasters',
-        'playlist_mincount': 3,
+        'playlist_mincount': 5,
         'info_dict': {
             'id': 'airdisasters',
             'title': 'Air Disasters',
@@ -1847,7 +1847,7 @@ class YoutubeShowIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(
-            url, playlist_id, 'Downloading show webpage')
+            'http://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
         # There's one playlist for each season of the show
         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
         self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))

From 4003bd82b099872d602713824f0a36957a4f1266 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 7 Sep 2015 12:56:16 +0200
Subject: [PATCH 1274/2145] [youtube:shows] Use https

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d7b90016..97ce36550 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1835,7 +1835,7 @@ class YoutubeShowIE(InfoExtractor):
     _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
     IE_NAME = 'youtube:show'
     _TESTS = [{
-        'url': 'http://www.youtube.com/show/airdisasters',
+        'url': 'https://www.youtube.com/show/airdisasters',
         'playlist_mincount': 5,
         'info_dict': {
             'id': 'airdisasters',
@@ -1847,7 +1847,7 @@ class YoutubeShowIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(
-            'http://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
+            'https://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
         # There's one playlist for each season of the show
         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
         self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))

From b1b7d1ffbac7150416e3cfcb3c2fb4ddddb9a6a4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 12:49:59 +0100
Subject: [PATCH 1275/2145] [wimp] fix tests info

---
 youtube_dl/extractor/wimp.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index f69d46a28..e4f50e64c 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -1,40 +1,33 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 
 
 class WimpIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wimp\.com/([^/]+)/'
+    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)/'
     _TESTS = [{
         'url': 'http://www.wimp.com/maruexhausted/',
-        'md5': 'f1acced123ecb28d9bb79f2479f2b6a1',
+        'md5': 'ee21217ffd66d058e8b16be340b74883',
         'info_dict': {
             'id': 'maruexhausted',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Maru is exhausted.',
             'description': 'md5:57e099e857c0a4ea312542b684a869b8',
         }
     }, {
-        # youtube video
         'url': 'http://www.wimp.com/clowncar/',
+        'md5': '4e2986c793694b55b37cf92521d12bb4',
         'info_dict': {
-            'id': 'cG4CEr2aiSg',
+            'id': 'clowncar',
             'ext': 'mp4',
-            'title': 'Basset hound clown car...incredible!',
-            'description': 'md5:8d228485e0719898c017203f900b3a35',
-            'uploader': 'Gretchen Hoey',
-            'uploader_id': 'gretchenandjeff1',
-            'upload_date': '20140303',
+            'title': 'It\'s like a clown car.',
+            'description': 'md5:0e56db1370a6e49c5c1d19124c0d2fb2',
         },
-        'add_ie': ['Youtube'],
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
             [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],

From 64997815c49a79c3d91e65198cc79d13bed234e9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 20:53:37 +0100
Subject: [PATCH 1276/2145] [ndr] add support for embeds

---
 youtube_dl/extractor/__init__.py |  2 +
 youtube_dl/extractor/ndr.py      | 84 +++++++++++++++++++++++++++-----
 2 files changed, 73 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..84830c885 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -368,6 +368,8 @@ from .nbc import (
 from .ndr import (
     NDRIE,
     NJoyIE,
+    NDREmbedIE,
+    NJoyEmbedIE,
 )
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 77b7e5eb8..aca85825d 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -12,21 +12,18 @@ preferences = {'xl': 4, 'l': 3, 'm': 2, 's': 1, 'xs': 0,}
 
 
 class NDRBaseIE(InfoExtractor):
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, 'Downloading page')
 
+    def extract_video_info(self, playlist, video_id):
         formats = []
-        objetType = json_data.get('config').get('objectType')
-        if objetType == 'video':
-            for key, f in json_data.get('playlist').items():
+        streamType = playlist.get('config').get('streamType')
+        if streamType == 'httpVideo':
+            for key, f in playlist.items():
                 if key != 'config':
                     src = f['src']
                     if '.f4m' in src:
                         formats.extend(self._extract_f4m_formats(src, video_id))
                     elif '.m3u8' in src:
-                        formats.extend(self._extract_m3u8_formats(src, video_id))
+                        formats.extend(self._extract_m3u8_formats(src, video_id, fatal=False))
                     else:
                         quality = f.get('quality')
                         formats.append({
@@ -34,20 +31,20 @@ class NDRBaseIE(InfoExtractor):
                             'format_id': quality,
                             'preference': preferences.get(quality),
                         })
-        elif objetType == 'audio':
-            for key, f in json_data.get('playlist').items():
+        elif streamType == 'httpAudio':
+            for key, f in playlist.items():
                 if key != 'config':
                     formats.append({
                         'url': f['src'],
                         'format_id': 'mp3',
-                        
+                        'vcodec': 'none',
                     })
         else:
             raise ExtractorError('No media links available for %s' % video_id)
 
         self._sort_formats(formats)
 
-        config = json_data.get('playlist').get('config')
+        config = playlist.get('config')
 
         title = config['title']
         duration = int_or_none(config.get('duration'))
@@ -65,6 +62,25 @@ class NDRBaseIE(InfoExtractor):
             'formats': formats,
         }
 
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
+
+        if not json_data:
+            webpage = self._download_webpage(url, video_id)
+            embed_url = self._html_search_regex(r'<iframe[^>]+id="pp_\w+"[^>]+src="(/.*)"', webpage, 'embed url', None, False)
+            if not embed_url:
+                embed_url = self._html_search_meta('embedURL', webpage, fatal=False)
+            if embed_url:
+                if embed_url.startswith('/'):
+                    return self.url_result('http://www.ndr.de%s' % embed_url, 'NDREmbed')
+                else:
+                    return self.url_result(embed_url, 'NDREmbed')
+            raise ExtractorError('No media links available for %s' % video_id)
+
+        return self.extract_video_info(json_data['playlist'], video_id)
+
 
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
@@ -116,9 +132,51 @@ class NJoyIE(NDRBaseIE):
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
         'md5': 'cb63be60cd6f9dd75218803146d8dc67',
         'info_dict': {
-            'id': '2480',
+            'id': 'comedycontest2480',
             'ext': 'mp4',
             'title': 'Benaissa beim NDR Comedy Contest',
             'duration': 654,
         }
     }
+
+
+class NDREmbedBaseIE(NDRBaseIE):
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
+        if not json_data:
+            raise ExtractorError('No media links available for %s' % video_id)
+        return self.extract_video_info(json_data['playlist'], video_id)
+
+
+class NDREmbedIE(NDREmbedBaseIE):
+    IE_NAME = 'ndr:embed'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>\w+)'
+
+    _TEST = {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': 'ndraktuell28488',
+            'ext': 'mp4',
+            'title': 'Norddeutschland begrüßt Flüchtlinge',
+            'duration': 132,
+        }
+    }
+
+
+class NJoyEmbedIE(NDREmbedBaseIE):
+    IE_NAME = 'N-JOY:embed'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)(?P<id>\w+)'
+
+    _TEST = {
+        'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': 'portraet374',
+            'ext': 'mp4',
+            'title': 'Viviane Andereggen - "Schuld um Schuld"',
+            'duration': 129,
+        }
+    }

From b88ebd472e555189cc0809f6497b931266733e1b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 21:25:32 +0100
Subject: [PATCH 1277/2145] [musicvault] remove extractor

---
 youtube_dl/extractor/musicvault.py | 63 ------------------------------
 1 file changed, 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/musicvault.py

diff --git a/youtube_dl/extractor/musicvault.py b/youtube_dl/extractor/musicvault.py
deleted file mode 100644
index 0e46ac7c1..000000000
--- a/youtube_dl/extractor/musicvault.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class MusicVaultIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.musicvault\.com/(?P<uploader_id>[^/?#]*)/video/(?P<display_id>[^/?#]*)_(?P<id>[0-9]+)\.html'
-    _TEST = {
-        'url': 'http://www.musicvault.com/the-allman-brothers-band/video/straight-from-the-heart_1010863.html',
-        'md5': '3adcbdb3dcc02d647539e53f284ba171',
-        'info_dict': {
-            'id': '1010863',
-            'ext': 'mp4',
-            'uploader_id': 'the-allman-brothers-band',
-            'title': 'Straight from the Heart',
-            'duration': 244,
-            'uploader': 'The Allman Brothers Band',
-            'thumbnail': 're:^https?://.*/thumbnail/.*',
-            'upload_date': '20131219',
-            'location': 'Capitol Theatre (Passaic, NJ)',
-            'description': 'Listen to The Allman Brothers Band perform Straight from the Heart at Capitol Theatre (Passaic, NJ) on Dec 16, 1981',
-            'timestamp': int,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id)
-
-        thumbnail = self._search_regex(
-            r'<meta itemprop="thumbnail" content="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-
-        data_div = self._search_regex(
-            r'(?s)<div class="data">(.*?)</div>', webpage, 'data fields')
-        uploader = self._html_search_regex(
-            r'<h1.*?>(.*?)</h1>', data_div, 'uploader', fatal=False)
-        title = self._html_search_regex(
-            r'<h2.*?>(.*?)</h2>', data_div, 'title')
-        location = self._html_search_regex(
-            r'<h4.*?>(.*?)</h4>', data_div, 'location', fatal=False)
-
-        kaltura_id = self._search_regex(
-            r'<div id="video-detail-player" data-kaltura-id="([^"]+)"',
-            webpage, 'kaltura ID')
-        wid = self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid')
-
-        return {
-            'id': mobj.group('id'),
-            '_type': 'url_transparent',
-            'url': 'kaltura:%s:%s' % (wid, kaltura_id),
-            'ie_key': 'Kaltura',
-            'display_id': display_id,
-            'uploader_id': mobj.group('uploader_id'),
-            'thumbnail': thumbnail,
-            'description': self._html_search_meta('description', webpage),
-            'location': location,
-            'title': title,
-            'uploader': uploader,
-        }

From 7a459170fa476eb5b5dde790241b4e961e0d6a22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Sep 2015 05:40:00 +0600
Subject: [PATCH 1278/2145] [__init__] Remove musicvault

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..57f55b479 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -347,7 +347,6 @@ from .mtv import (
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
-from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
 from .mwave import MwaveIE
 from .myspace import MySpaceIE, MySpaceAlbumIE

From 12810c9cd3c2a5a50ca0ea309d27af2decc8b263 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Sep 2015 14:11:20 +0600
Subject: [PATCH 1279/2145] [crunchyroll] Force Accept-Language to any for all
 requests (Closes #6797)

---
 youtube_dl/extractor/crunchyroll.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index ce123482e..c5dbc682d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -31,7 +31,23 @@ from ..aes import (
 )
 
 
-class CrunchyrollIE(InfoExtractor):
+class CrunchyrollBaseIE(InfoExtractor):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
+        request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
+                   else compat_urllib_request.Request(url_or_request))
+        # Accept-Language must be set explicitly to accept any language to avoid issues
+        # similar to https://github.com/rg3/youtube-dl/issues/6797.
+        # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
+        # should be imposed or not (from what I can see it just takes the first language
+        # ignoring the priority and requires it to correspond the IP). By the way this causes
+        # Crunchyroll to not work in georestriction cases in some browsers that don't place
+        # the locale lang first in header. However allowing any language seems to workaround the issue.
+        request.add_header('Accept-Language', '*')
+        return super(CrunchyrollBaseIE, self)._download_webpage(
+            request, video_id, note, errnote, fatal, tries, timeout, encoding)
+
+
+class CrunchyrollIE(CrunchyrollBaseIE):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
@@ -330,7 +346,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         }
 
 
-class CrunchyrollShowPlaylistIE(InfoExtractor):
+class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     IE_NAME = "crunchyroll:playlist"
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
 

From 47004d9579839137bff17949f38db2844829bbd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Sep 2015 14:37:53 +0600
Subject: [PATCH 1280/2145] [crunchyroll] Fix uploader and upload date
 extraction

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c5dbc682d..95952bc29 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -275,10 +275,14 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
         if not video_description:
             video_description = None
-        video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
+        video_upload_date = self._html_search_regex(
+            [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
+            webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
         if video_upload_date:
             video_upload_date = unified_strdate(video_upload_date)
-        video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
+        video_uploader = self._html_search_regex(
+            r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', webpage,
+            'video_uploader', fatal=False)
 
         playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = compat_urllib_request.Request(playerdata_url)

From b58a22b9632a226fc453bb52f195e7577476f35e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 15:37:20 +0100
Subject: [PATCH 1281/2145] [ndr] make NDREmbed and NJoyEmbed _VALID_URL regex
 unique

---
 youtube_dl/extractor/ndr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index aca85825d..2a73650d4 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -152,7 +152,7 @@ class NDREmbedBaseIE(NDRBaseIE):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>\w+)'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
 
     _TEST = {
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
@@ -168,7 +168,7 @@ class NDREmbedIE(NDREmbedBaseIE):
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'N-JOY:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)(?P<id>\w+)'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
 
     _TEST = {
         'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html',

From 376e1ad081abbeb234419c08be121fe81eb9b51d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 15:58:32 +0100
Subject: [PATCH 1282/2145] [ndr] use utils.qualites

---
 youtube_dl/extractor/ndr.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 2a73650d4..87f3edbbe 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -5,10 +5,11 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
+    qualities,
 )
 
 
-preferences = {'xl': 4, 'l': 3, 'm': 2, 's': 1, 'xs': 0,}
+preference = qualities(['xs', 's', 'm','l', 'xl'])
 
 
 class NDRBaseIE(InfoExtractor):
@@ -29,7 +30,7 @@ class NDRBaseIE(InfoExtractor):
                         formats.append({
                             'url': src,
                             'format_id': quality,
-                            'preference': preferences.get(quality),
+                            'preference': preference(quality),
                         })
         elif streamType == 'httpAudio':
             for key, f in playlist.items():
@@ -51,7 +52,7 @@ class NDRBaseIE(InfoExtractor):
         thumbnails = [{
             'id': thumbnail.get('quality'),
             'url': thumbnail.get('src'),
-            'preference': preferences.get(thumbnail.get('quality'))
+            'preference': preference(thumbnail.get('quality'))
         } for thumbnail in config.get('poster').values()]
 
         return {

From 75bb5c7028bd9072bd03371c2d337932ec832642 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 20:12:30 +0100
Subject: [PATCH 1283/2145] [xuite] fix test md5

---
 youtube_dl/extractor/xuite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 5aac8adb3..8bbac54e2 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -19,7 +19,7 @@ class XuiteIE(InfoExtractor):
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
-        'md5': '63a42c705772aa53fd4c1a0027f86adf',
+        'md5': 'e79284c87b371424885448d11f6398c8',
         'info_dict': {
             'id': '3860914',
             'ext': 'mp3',

From 10723362492845d25d4ad996dd209dafbce7eb04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 20:40:23 +0100
Subject: [PATCH 1284/2145] [washingtonpost] fix test info

---
 youtube_dl/extractor/washingtonpost.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 72eb010f8..ec8b99998 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -19,25 +19,25 @@ class WashingtonPostIE(InfoExtractor):
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
-            'md5': '79132cc09ec5309fa590ae46e4cc31bc',
+            'md5': 'b9be794ceb56c7267d410a13f99d801a',
             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
                 'title': 'Breaking Points: The Paper Mine',
-                'duration': 1287,
+                'duration': 1290,
                 'description': 'Overly complicated paper pushing is nothing new to government bureaucracy. But the way federal retirement applications are filed may be the most outdated. David Fahrenthold explains.',
                 'uploader': 'The Washington Post',
                 'timestamp': 1395527908,
                 'upload_date': '20140322',
             },
         }, {
-            'md5': 'e1d5734c06865cc504ad99dc2de0d443',
+            'md5': '1fff6a689d8770966df78c8cb6c8c17c',
             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
                 'title': 'The town bureaucracy sustains',
                 'description': 'Underneath the friendly town of Boyers is a sea of government paperwork. In a disused limestone mine, hundreds of locals now track, file and process retirement applications for the federal government. We set out to find out what it\'s like to do paperwork 230 feet underground.',
-                'duration': 2217,
+                'duration': 2220,
                 'timestamp': 1395528005,
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',

From 7b4137c351222a94f46f854bf490a299e4124acc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 9 Sep 2015 10:42:47 +0100
Subject: [PATCH 1285/2145] [fktv] fix info extraction

---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/fktv.py     | 78 ++++++++------------------------
 2 files changed, 20 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..f8d4c8462 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -170,10 +170,7 @@ from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
-from .fktv import (
-    FKTVIE,
-    FKTVPosteckeIE,
-)
+from .fktv import FKTVIE
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 190d9f9ad..c2aa23aa2 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
 import re
-import random
-import json
 
 from .common import InfoExtractor
 from ..utils import (
-    get_element_by_id,
     clean_html,
+    determine_ext,
 )
 
 
@@ -17,66 +15,28 @@ class FKTVIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
+        'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79',
         'info_dict': {
-            'id': '00011',
-            'ext': 'flv',
+            'id': '1',
+            'ext': 'mp4',
             'title': 'Folge 1 vom 10. April 2007',
-            'description': 'md5:fb4818139c7cfe6907d4b83412a6864f',
         },
     }
 
     def _real_extract(self, url):
-        episode = int(self._match_id(url))
+        episode = self._match_id(url)
 
-        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%s.jpg' % episode
-        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/Start' % episode,
-                                               episode)
-        playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
-                                      'playlist', flags=re.DOTALL)
-        files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
-
-        videos = []
-        for i, _ in enumerate(files, 1):
-            video_id = '%04d%d' % (episode, i)
-            video_url = 'http://fernsehkritik.tv/js/directme.php?file=%s%s.flv' % (episode, '' if i == 1 else '-%d' % i)
-            videos.append({
-                'ext': 'flv',
-                'id': video_id,
-                'url': video_url,
-                'title': clean_html(get_element_by_id('eptitle', start_webpage)),
-                'description': clean_html(get_element_by_id('contentlist', start_webpage)),
-                'thumbnail': video_thumbnail
-            })
-        return {
-            '_type': 'multi_video',
-            'entries': videos,
-            'id': 'folge-%s' % episode,
-        }
-
-
-class FKTVPosteckeIE(InfoExtractor):
-    IE_NAME = 'fernsehkritik.tv:postecke'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
-    _TEST = {
-        'url': 'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
-        'md5': '262f0adbac80317412f7e57b4808e5c4',
-        'info_dict': {
-            'id': '0120',
-            'ext': 'flv',
-            'title': 'Postecke 120',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = int(mobj.group('ep'))
-
-        server = random.randint(2, 4)
-        video_id = '%04d' % episode
-        video_url = 'http://dl%d.fernsehkritik.tv/postecke/postecke%d.flv' % (server, episode)
-        video_title = 'Postecke %d' % episode
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': video_title,
-        }
+        webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
+        title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
+        matchs = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
+        if matchs:
+            poster, sources = matchs.groups()
+            urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
+            if sources:
+                formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
+                return {
+                    'id': episode,
+                    'title': title,
+                    'formats': formats,
+                    'thumbnail': poster,
+                }

From d9c19db340c568db1afcd6d85ab5b9c4681dd7df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Wed, 9 Sep 2015 17:02:28 +0200
Subject: [PATCH 1286/2145] Extract all vid.me formats

---
 youtube_dl/extractor/vidme.py | 82 ++++++++++++++++++++++-------------
 1 file changed, 52 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 157bb74fe..7f44948e9 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .common import InfoExtractor, ExtractorError
 from ..utils import (
     int_or_none,
     float_or_none,
-    str_to_int,
+    parse_iso8601,
 )
 
 
@@ -12,7 +12,7 @@ class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
-        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
+        'md5': 'c62f1156138dc3323902188c5b5a8bd6',
         'info_dict': {
             'id': 'QNB',
             'ext': 'mp4',
@@ -24,6 +24,26 @@ class VidmeIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
+            'comment_count': int,
+        },
+    }, {
+        'url': 'https://vid.me/Gc6M',
+        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
+        'info_dict': {
+            'id': 'Gc6M',
+            'ext': 'mp4',
+            'title': 'O Mere Dil ke chain - Arnav and Khushi VM',
+            'duration': 223.72,
+            'timestamp': 1441211642,
+            'upload_date': '20150902',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         # tests uploader field
@@ -40,6 +60,7 @@ class VidmeIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
+            'comment_count': int,
         },
         'params': {
             'skip_download': True,
@@ -51,45 +72,46 @@ class VidmeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        url = url.replace('vid.me/e/', 'vid.me/')
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        api_url = 'https://api.vid.me/videoByUrl/' + video_id
+        data = self._download_json(api_url, video_id)
 
-        video_url = self._html_search_regex(
-            r'<source src="([^"]+)"', webpage, 'video URL')
+        video_data = data.get('video')
+        if video_data is None:
+            raise ExtractorError('Could not extract the vid.me video data')
 
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage, default='')
-        thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = int_or_none(self._og_search_property(
-            'updated_time', webpage, fatal=False))
-        width = int_or_none(self._og_search_property(
-            'video:width', webpage, fatal=False))
-        height = int_or_none(self._og_search_property(
-            'video:height', webpage, fatal=False))
-        duration = float_or_none(self._html_search_regex(
-            r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
-        view_count = str_to_int(self._html_search_regex(
-            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?',
-            webpage, 'view count', fatal=False))
-        like_count = str_to_int(self._html_search_regex(
-            r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
-            webpage, 'like count', fatal=False))
-        uploader = self._html_search_regex(
-            'class="video_author_username"[^>]*>([^<]+)',
-            webpage, 'uploader', default=None)
+        title = video_data.get('title')
+        description = video_data.get('description')
+        thumbnail = video_data.get('thumbnail_url')
+        timestamp = parse_iso8601(video_data.get('date_created'), ' ')
+        duration = float_or_none(video_data.get('duration'))
+        view_count = int_or_none(video_data.get('view_count'))
+        like_count = int_or_none(video_data.get('likes_count'))
+        comment_count = int_or_none(video_data.get('comment_count'))
+
+        uploader = None
+        user_data = video_data.get('user')
+        if user_data is not None:
+            uploader = user_data.get('username')
+
+        formats = [{
+            'format_id': format['type'],
+            'url': format['uri'],
+            'width': int_or_none(format['width']),
+            'height': int_or_none(format['height']),
+        } for format in video_data.get('formats', [])]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
-            'width': width,
-            'height': height,
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
+            'comment_count': comment_count,
             'uploader': uploader,
+            'formats': formats,
         }

From 482aa3feccd70e773346bfd4598cba4ea6e80621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 22:04:05 +0600
Subject: [PATCH 1287/2145] [vidme] Simplify, make more robust, extract more
 metadata and capture errors (Closes #6812)

---
 youtube_dl/extractor/vidme.py | 100 +++++++++++++++++++++++-----------
 1 file changed, 67 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 7f44948e9..9a794e609 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,7 +1,9 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor, ExtractorError
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
+    ExtractorError,
     int_or_none,
     float_or_none,
     parse_iso8601,
@@ -18,10 +20,11 @@ class VidmeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Fishing for piranha - the easy way',
             'description': 'source: https://www.facebook.com/photo.php?v=312276045600871',
-            'duration': 119.92,
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1406313244,
             'upload_date': '20140725',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'age_limit': 0,
+            'duration': 119.92,
             'view_count': int,
             'like_count': int,
             'comment_count': int,
@@ -33,14 +36,16 @@ class VidmeIE(InfoExtractor):
             'id': 'Gc6M',
             'ext': 'mp4',
             'title': 'O Mere Dil ke chain - Arnav and Khushi VM',
-            'duration': 223.72,
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1441211642,
             'upload_date': '20150902',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'SunshineM',
+            'uploader_id': '3552827',
+            'age_limit': 0,
+            'duration': 223.72,
             'view_count': int,
             'like_count': int,
             'comment_count': int,
-            'comment_count': int,
         },
         'params': {
             'skip_download': True,
@@ -53,11 +58,13 @@ class VidmeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Carver',
             'description': 'md5:e9c24870018ae8113be936645b93ba3c',
-            'duration': 97.859999999999999,
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1433203629,
             'upload_date': '20150602',
             'uploader': 'Thomas',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader_id': '109747',
+            'age_limit': 0,
+            'duration': 97.859999999999999,
             'view_count': int,
             'like_count': int,
             'comment_count': int,
@@ -66,52 +73,79 @@ class VidmeIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
+        # nsfw test from http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
         'url': 'https://vid.me/e/Wmur',
-        'only_matching': True,
+        'info_dict': {
+            'id': 'Wmur',
+            'ext': 'mp4',
+            'title': 'naked smoking & stretching',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1430931613,
+            'upload_date': '20150506',
+            'uploader': 'naked-yogi',
+            'uploader_id': '1638622',
+            'age_limit': 18,
+            'duration': 653.26999999999998,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        api_url = 'https://api.vid.me/videoByUrl/' + video_id
-        data = self._download_json(api_url, video_id)
 
-        video_data = data.get('video')
-        if video_data is None:
-            raise ExtractorError('Could not extract the vid.me video data')
+        try:
+            response = self._download_json(
+                'https://api.vid.me/videoByUrl/%s' % video_id, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                response = self._parse_json(e.cause.read(), video_id)
+            else:
+                raise
 
-        title = video_data.get('title')
-        description = video_data.get('description')
-        thumbnail = video_data.get('thumbnail_url')
-        timestamp = parse_iso8601(video_data.get('date_created'), ' ')
-        duration = float_or_none(video_data.get('duration'))
-        view_count = int_or_none(video_data.get('view_count'))
-        like_count = int_or_none(video_data.get('likes_count'))
-        comment_count = int_or_none(video_data.get('comment_count'))
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        uploader = None
-        user_data = video_data.get('user')
-        if user_data is not None:
-            uploader = user_data.get('username')
+        video = response['video']
 
         formats = [{
-            'format_id': format['type'],
-            'url': format['uri'],
-            'width': int_or_none(format['width']),
-            'height': int_or_none(format['height']),
-        } for format in video_data.get('formats', [])]
+            'format_id': f.get('type'),
+            'url': f['uri'],
+            'width': int_or_none(f.get('width')),
+            'height': int_or_none(f.get('height')),
+        } for f in video.get('formats', []) if f.get('uri')]
         self._sort_formats(formats)
 
+        title = video['title']
+        description = video.get('description')
+        thumbnail = video.get('thumbnail_url')
+        timestamp = parse_iso8601(video.get('date_created'), ' ')
+        uploader = video.get('user', {}).get('username')
+        uploader_id = video.get('user', {}).get('user_id')
+        age_limit = 18 if video.get('nsfw') is True else 0
+        duration = float_or_none(video.get('duration'))
+        view_count = int_or_none(video.get('view_count'))
+        like_count = int_or_none(video.get('likes_count'))
+        comment_count = int_or_none(video.get('comment_count'))
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'age_limit': age_limit,
             'timestamp': timestamp,
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'comment_count': comment_count,
-            'uploader': uploader,
             'formats': formats,
         }

From 2f29b758e0a37e084bc4a76bc920919bf9d6b181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:41:44 +0600
Subject: [PATCH 1288/2145] Credit @lalinsky for vidme improvement (#6812)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d1693224e..8e42e1f5e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -140,3 +140,4 @@ Behrouz Abbasi
 ngld
 nyuszika7h
 Shaun Walbridge
+Lukáš Lalinský

From b17e7d9a9bac82485a2d1241e1e185593f1ac93a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:44:49 +0600
Subject: [PATCH 1289/2145] Credit @anssih for missing metadata extraction on
 youtube (#5993)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8e42e1f5e..2087e5141 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -140,4 +140,5 @@ Behrouz Abbasi
 ngld
 nyuszika7h
 Shaun Walbridge
+Anssi Hannula
 Lukáš Lalinský

From 73f536439ee70145237c72127262052206097e20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:49:36 +0600
Subject: [PATCH 1290/2145] Credit @reddraggone9 for fixing youtube 2-factor
 authentication (#6553)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2087e5141..901c1b263 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -140,5 +140,6 @@ Behrouz Abbasi
 ngld
 nyuszika7h
 Shaun Walbridge
+Lee Jenkins
 Anssi Hannula
 Lukáš Lalinský

From cc1ac110173f77a22c9becbee4cb6522087986a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:59:17 +0600
Subject: [PATCH 1291/2145] [vier:videos] Fix extraction with old approach
 (Closes #6806)

---
 youtube_dl/extractor/vier.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 15377097e..c76c20614 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 
@@ -91,31 +92,27 @@ class VierVideosIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         program = mobj.group('program')
 
-        webpage = self._download_webpage(url, program)
-
         page_id = mobj.group('page')
         if page_id:
             page_id = int(page_id)
             start_page = page_id
-            last_page = start_page + 1
             playlist_id = '%s-page%d' % (program, page_id)
         else:
             start_page = 0
-            last_page = int(self._search_regex(
-                r'videos\?page=(\d+)">laatste</a>',
-                webpage, 'last page', default=0)) + 1
             playlist_id = program
 
         entries = []
-        for current_page_id in range(start_page, last_page):
+        for current_page_id in itertools.count(start_page):
             current_page = self._download_webpage(
                 'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id),
                 program,
-                'Downloading page %d' % (current_page_id + 1)) if current_page_id != page_id else webpage
+                'Downloading page %d' % (current_page_id + 1))
             page_entries = [
                 self.url_result('http://www.vier.be' + video_url, 'Vier')
                 for video_url in re.findall(
                     r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
             entries.extend(page_entries)
+            if page_id or '>Meer<' not in current_page:
+                break
 
         return self.playlist_result(entries, playlist_id)

From d85187eb747b9d32d44ce4f31985432288f4a4f2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 9 Sep 2015 21:20:01 +0200
Subject: [PATCH 1292/2145] release 2015.09.09

---
 docs/supportedsites.md | 4 ++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 04b9959ac..66091e6be 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -195,7 +195,7 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net
+ - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com
  - **Goshgay**
  - **Groupon**
  - **Hark**
@@ -309,7 +309,6 @@
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
  - **MusicPlayOn**
- - **MusicVault**
  - **muzu.tv**
  - **Mwave**
  - **MySpace**
@@ -633,6 +632,7 @@
  - **vine:user**
  - **vk**: VK
  - **vk:uservideos**: VK - User's Videos
+ - **vlive**
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6bc689b75..0cc7411f2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.03'
+__version__ = '2015.09.09'

From 133a2b4ac21e041d7ed66c1d0fa00f7dc5f757af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Sep 2015 02:41:30 +0600
Subject: [PATCH 1293/2145] [downloader/hls] Properly close stream

---
 youtube_dl/downloader/hls.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 71aafdc73..b2436e732 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -92,6 +92,7 @@ class NativeHlsFD(FragmentFD):
                 return False
             down, frag_sanitized = sanitize_open(frag_filename, 'rb')
             ctx['dest_stream'].write(down.read())
+            down.close()
             frags_filenames.append(frag_sanitized)
 
         self._finish_frag_download(ctx)

From 364ca0582eb1506af049c0e17595509c6bc0e00c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Sep 2015 02:49:33 +0600
Subject: [PATCH 1294/2145] [yahoo] Process query result embedded in webpage

---
 youtube_dl/extractor/yahoo.py | 63 ++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b8579b573..1d9b98750 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -144,6 +144,20 @@ class YahooIE(InfoExtractor):
         }, {
             'url': 'https://tw.news.yahoo.com/-100120367.html',
             'only_matching': True,
+        }, {
+            # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
+            'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
+            'info_dict': {
+                'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
+                'ext': 'mp4',
+                'title': 'Communitary - Community Episode 1: Ladders',
+                'description': 'md5:8fc39608213295748e1e289807838c97',
+                'duration': 1646,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         }
     ]
 
@@ -171,6 +185,19 @@ class YahooIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Query result is often embedded in webpage as JSON. Sometimes explicit requests
+        # to video API results in a failure with geo restriction reason therefore using
+        # embedded query result when present sounds reasonable.
+        config_json = self._search_regex(
+            r'window\.Af\.bootstrap\[[^\]]+\]\s*=\s*({.*?"applet_type"\s*:\s*"td-applet-videoplayer".*?});(?:</script>|$)',
+            webpage, 'videoplayer applet', default=None)
+        if config_json:
+            config = self._parse_json(config_json, display_id, fatal=False)
+            if config:
+                sapi = config.get('models', {}).get('applet_model', {}).get('data', {}).get('sapi')
+                if sapi:
+                    return self._extract_info(display_id, sapi, webpage)
+
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
             default=None)
@@ -190,22 +217,10 @@ class YahooIE(InfoExtractor):
             video_id = info['id']
         return self._get_info(video_id, display_id, webpage)
 
-    def _get_info(self, video_id, display_id, webpage):
-        region = self._search_regex(
-            r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
-            webpage, 'region', fatal=False, default='US')
-        data = compat_urllib_parse.urlencode({
-            'protocol': 'http',
-            'region': region,
-        })
-        query_url = (
-            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
-            '{id}?{data}'.format(id=video_id, data=data))
-        query_result = self._download_json(
-            query_url, display_id, 'Downloading video info')
-
-        info = query_result['query']['results']['mediaObj'][0]
+    def _extract_info(self, display_id, query, webpage):
+        info = query['query']['results']['mediaObj'][0]
         meta = info.get('meta')
+        video_id = info.get('id')
 
         if not meta:
             msg = info['status'].get('msg')
@@ -231,6 +246,9 @@ class YahooIE(InfoExtractor):
                     'ext': 'flv',
                 })
             else:
+                if s.get('format') == 'm3u8_playlist':
+                    format_info['protocol'] = 'm3u8_native'
+                    format_info['ext'] = 'mp4'
                 format_url = compat_urlparse.urljoin(host, path)
                 format_info['url'] = format_url
             formats.append(format_info)
@@ -264,6 +282,21 @@ class YahooIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
+    def _get_info(self, video_id, display_id, webpage):
+        region = self._search_regex(
+            r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
+            webpage, 'region', fatal=False, default='US')
+        data = compat_urllib_parse.urlencode({
+            'protocol': 'http',
+            'region': region,
+        })
+        query_url = (
+            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
+            '{id}?{data}'.format(id=video_id, data=data))
+        query_result = self._download_json(
+            query_url, display_id, 'Downloading video info')
+        return self._extract_info(display_id, query_result, webpage)
+
 
 class YahooSearchIE(SearchInfoExtractor):
     IE_DESC = 'Yahoo screen search'

From 1721fef28b89ac4264db978ab7fee3b4dd154056 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Sep 2015 02:58:40 +0600
Subject: [PATCH 1295/2145] [yahoo] Fix test

---
 youtube_dl/extractor/yahoo.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 1d9b98750..fca5ddc69 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -147,6 +147,7 @@ class YahooIE(InfoExtractor):
         }, {
             # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
             'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
+            'md5': '4fbafb9c9b6f07aa8f870629f6671b35',
             'info_dict': {
                 'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
                 'ext': 'mp4',
@@ -154,10 +155,6 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:8fc39608213295748e1e289807838c97',
                 'duration': 1646,
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
         }
     ]
 

From 689fb748ee1ba8e61f99d21a3bcb1bc83b708649 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Sep 2015 04:44:17 +0100
Subject: [PATCH 1296/2145] [utlis] add extract_attributes for extracting html
 tags attributes

---
 youtube_dl/utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 206dd56bc..bcebf9cc5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -248,6 +248,14 @@ def get_element_by_attribute(attribute, value, html):
     return unescapeHTML(res)
 
 
+def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s*["\']([^"\']+)["\']'):
+    attributes = re.findall(attributes_regex, attributes_str)
+    attributes_dict = {}
+    if attributes:
+        attributes_dict = {attribute_name: attribute_value for (attribute_name, attribute_value) in attributes}
+    return attributes_dict
+
+
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 

From ed1269000f24a6ddc683a295ff402ef3ded5c4fb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Sep 2015 04:46:21 +0100
Subject: [PATCH 1297/2145] [brightcove] add support for brightcove in page
 embed(fixes #6824)

---
 youtube_dl/extractor/__init__.py   |  5 +-
 youtube_dl/extractor/brightcove.py | 92 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py    | 21 ++++++-
 3 files changed, 116 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..fcd9edec3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -59,7 +59,10 @@ from .bloomberg import BloombergIE
 from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
-from .brightcove import BrightcoveIE
+from .brightcove import (
+    BrightcoveIE,
+    BrightcoveInPageEmbedIE,
+)
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4721c2293..a07c0888f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -22,6 +22,10 @@ from ..utils import (
     fix_xml_ampersands,
     unescapeHTML,
     unsmuggle_url,
+    js_to_json,
+    int_or_none,
+    parse_iso8601,
+    extract_attributes,
 )
 
 
@@ -346,3 +350,91 @@ class BrightcoveIE(InfoExtractor):
         if 'url' not in info and not info.get('formats'):
             raise ExtractorError('Unable to extract video url for %s' % info['id'])
         return info
+
+
+class BrightcoveInPageEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
+    TEST = {
+        'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
+        'info_dict': {
+            'id': '4463358922001',
+            'ext': 'flv',
+            'title': 'Meet the man behind Popcorn Time',
+            'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+            'duration': 165768,
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        video_attributes = re.search(r'(?s)<video([^>]*)>.*?</(?:video|audio)>', webpage)
+        if video_attributes:
+            video_attributes = extract_attributes(video_attributes.group(), r'(?s)\s*data-(account|video-id|playlist-id|policy-key|player|embed)\s*=\s*["\']([^"\']+)["\']')
+            account_id = video_attributes.get('account')
+            player_id = video_attributes.get('player')
+            embed = video_attributes.get('embed')
+            video_id = video_attributes.get('video-id')
+            if account_id and player_id and embed and video_id:
+                return 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (account_id, player_id, embed, video_id)
+        return None
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        account_id, player_id, embed, video_id = mobj.groups()
+
+        webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
+
+        catalog = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r'catalog\(({[^}]+})\);',
+                    webpage,
+                    'catalog'
+                )
+            ),
+            video_id
+        )
+        policy_key = catalog['policyKey']
+
+        req = compat_urllib_request.Request(
+            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id),
+            headers={'Accept': 'application/json;pk=%s' % policy_key})
+        json_data = self._download_json(req, video_id)
+
+        title = json_data['name']
+        description = json_data.get('description')
+        thumbnail = json_data.get('name')
+        timestamp = parse_iso8601(json_data.get('published_at'))
+        duration = int_or_none(json_data.get('duration'))
+
+        formats = []
+        for source in json_data.get('sources'):
+            source_type = source.get('type')
+            if source_type == 'application/x-mpegURL':
+                formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
+            else:
+                src = source.get('src')
+                if src:
+                    formats.append({
+                        'url': src,
+                        'abr': source.get('avg_bitrate'),
+                        'width': int_or_none(source.get('width')),
+                        'height': int_or_none(source.get('height')),
+                        'filesize': source.get('size'),
+                        'container': source.get('container'),
+                        'vcodec': source.get('container'),
+                    })
+                else:
+                    formats.extend(self._extract_f4m_formats(source.get('streaming_src'), video_id))
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec748ed9f..7a3a7f66b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,7 +29,10 @@ from ..utils import (
     url_basename,
     xpath_text,
 )
-from .brightcove import BrightcoveIE
+from .brightcove import (
+    BrightcoveIE,
+    BrightcoveInPageEmbedIE,
+)
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
@@ -1012,6 +1015,17 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'cinemasnob',
             },
+        },
+        # BrightcoveInPageEmbed embed
+        {
+            'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
+            'info_dict': {
+                'id': '4238694884001',
+                'ext': 'flv',
+                'title': 'Tabletop: Dread, Last Thoughts',
+                'description': 'Tabletop: Dread, Last Thoughts',
+                'duration': 51690,
+            },
         }
     ]
 
@@ -1288,6 +1302,11 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
+        # Look for Brightcove In Page Embed:
+        brightcove_in_page_embed_url = BrightcoveInPageEmbedIE._extract_url(webpage)
+        if brightcove_in_page_embed_url:
+            return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
+
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',

From 73eb13dfc74132b8f0e5c1ac1ea75f66e0aca6bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Sep 2015 20:43:05 +0600
Subject: [PATCH 1298/2145] [extractor/common] Case insensitive inputs
 extraction

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5eeeda08d..835f6f368 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -732,7 +732,7 @@ class InfoExtractor(object):
     @staticmethod
     def _hidden_inputs(html):
         hidden_inputs = {}
-        for input in re.findall(r'<input([^>]+)>', html):
+        for input in re.findall(r'(?i)<input([^>]+)>', html):
             if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
                 continue
             name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
@@ -746,7 +746,7 @@ class InfoExtractor(object):
 
     def _form_hidden_inputs(self, form_id, html):
         form = self._search_regex(
-            r'(?s)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
+            r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
             html, '%s form' % form_id, group='form')
         return self._hidden_inputs(form)
 

From 586f1cc532d167c28e733779cbf132b94d8f76e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Sep 2015 21:07:32 +0600
Subject: [PATCH 1299/2145] [extractor/common] Skip html comment tags (Closes
 #6822)

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 835f6f368..d694e818e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -731,6 +731,7 @@ class InfoExtractor(object):
 
     @staticmethod
     def _hidden_inputs(html):
+        html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
         hidden_inputs = {}
         for input in re.findall(r'(?i)<input([^>]+)>', html):
             if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):

From 60ed60353b9ca57e8181f0b14d525ce487e673ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Sep 2015 20:34:48 +0600
Subject: [PATCH 1300/2145] [openfilm] Remove extractor

OpenFilm has been shut down
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/openfilm.py | 70 --------------------------------
 2 files changed, 71 deletions(-)
 delete mode 100644 youtube_dl/extractor/openfilm.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..2e7272931 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,7 +432,6 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
-from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
deleted file mode 100644
index d2ceedd01..000000000
--- a/youtube_dl/extractor/openfilm.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote_plus
-from ..utils import (
-    parse_iso8601,
-    parse_age_limit,
-    int_or_none,
-)
-
-
-class OpenFilmIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)openfilm\.com/videos/(?P<id>.+)'
-    _TEST = {
-        'url': 'http://www.openfilm.com/videos/human-resources-remastered',
-        'md5': '42bcd88c2f3ec13b65edf0f8ad1cac37',
-        'info_dict': {
-            'id': '32736',
-            'display_id': 'human-resources-remastered',
-            'ext': 'mp4',
-            'title': 'Human Resources (Remastered)',
-            'description': 'Social Engineering in the 20th Century.',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 7164,
-            'timestamp': 1334756988,
-            'upload_date': '20120418',
-            'uploader_id': '41117',
-            'view_count': int,
-            'age_limit': 0,
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        player = compat_urllib_parse_unquote_plus(
-            self._og_search_video_url(webpage))
-
-        video = json.loads(self._search_regex(
-            r'\bp=({.+?})(?:&|$)', player, 'video JSON'))
-
-        video_url = '%s1.mp4' % video['location']
-        video_id = video.get('video_id')
-        display_id = video.get('alias') or display_id
-        title = video.get('title')
-        description = video.get('description')
-        thumbnail = video.get('main_thumb')
-        duration = int_or_none(video.get('duration'))
-        timestamp = parse_iso8601(video.get('dt_published'), ' ')
-        uploader_id = video.get('user_id')
-        view_count = int_or_none(video.get('views_count'))
-        age_limit = parse_age_limit(video.get('age_limit'))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'uploader_id': uploader_id,
-            'view_count': view_count,
-            'age_limit': age_limit,
-        }

From 41ebd6530b124b9265a3df9d7d09aef02041b088 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:42:57 +0800
Subject: [PATCH 1301/2145] [tudou] Add the test case (#6273)

---
 youtube_dl/extractor/tudou.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index e800477e2..950c42afb 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -27,6 +27,9 @@ class TudouIE(InfoExtractor):
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tudou.com/albumplay/cJAHGih4yYg.html',
+        'only_matching': True,
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'

From 94e507aea798dac6974237cc44257dda45d5fa5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:45:09 +0800
Subject: [PATCH 1302/2145] [tudou] A more comprehensive _VALID_URL

---
 youtube_dl/extractor/tudou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 950c42afb..68712cb4a 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/?.*/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/([^/]+/)*(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',

From 141ba36996f77a420df69903a59792f6f93ae314 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:51:49 +0800
Subject: [PATCH 1303/2145] [tudou] Modernize

---
 youtube_dl/extractor/tudou.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 68712cb4a..c9d80a7ef 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -2,9 +2,6 @@
 
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
@@ -46,13 +43,10 @@ class TudouIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage)
-        if m and m.group(1):
-            return {
-                '_type': 'url',
-                'url': 'youku:' + m.group(1),
-                'ie_key': 'Youku'
-            }
+        youku_vcode = self._search_regex(
+            r'vcode:\s*[\'"](.+?)[\'"]', webpage, 'youku vcode', default=None)
+        if youku_vcode:
+            return self.url_result('youku:' + youku_vcode, ie='Youku')
 
         title = self._search_regex(
             r",kw:\s*['\"](.+?)[\"']", webpage, 'title')
@@ -63,8 +57,8 @@ class TudouIE(InfoExtractor):
             r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
             webpage, 'player URL', default=self._PLAYER_URL)
 
-        segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
-        segments = json.loads(segs_json)
+        segments = self._parse_json(self._search_regex(
+            r'segs: \'(.*)\'', webpage, 'segments'), video_id)
         # It looks like the keys are the arguments that have to be passed as
         # the hd field in the request url, we pick the higher
         # Also, filter non-number qualities (see issue #3643).

From aab135516b288f24c55178b024976fd3e130c7b8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:52:51 +0800
Subject: [PATCH 1304/2145] [tudou] Avoid shadowing builtin names

---
 youtube_dl/extractor/tudou.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c9d80a7ef..6116b209d 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -31,11 +31,11 @@ class TudouIE(InfoExtractor):
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
 
-    def _url_for_id(self, id, quality=None):
-        info_url = "http://v2.tudou.com/f?id=" + str(id)
+    def _url_for_id(self, video_id, quality=None):
+        info_url = "http://v2.tudou.com/f?id=" + str(video_id)
         if quality:
             info_url += '&hd' + quality
-        webpage = self._download_webpage(info_url, id, "Opening the info webpage")
+        webpage = self._download_webpage(info_url, video_id, "Opening the info webpage")
         final_url = self._html_search_regex('>(.+?)</f>', webpage, 'video url')
         return final_url
 

From 87813a857009dc3c3dfcc421679e5e806d363863 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 02:36:51 +0800
Subject: [PATCH 1305/2145] [tudou] Use _download_xml

---
 youtube_dl/extractor/tudou.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 6116b209d..3b993192c 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -35,8 +35,8 @@ class TudouIE(InfoExtractor):
         info_url = "http://v2.tudou.com/f?id=" + str(video_id)
         if quality:
             info_url += '&hd' + quality
-        webpage = self._download_webpage(info_url, video_id, "Opening the info webpage")
-        final_url = self._html_search_regex('>(.+?)</f>', webpage, 'video url')
+        xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")
+        final_url = xml_data.text
         return final_url
 
     def _real_extract(self, url):

From 349b3a2ea0d6c264facacd92508516e8530108b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 02:51:20 +0800
Subject: [PATCH 1306/2145] [tudou] Improve regexs

---
 youtube_dl/extractor/tudou.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 3b993192c..53ba8511f 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -44,21 +44,21 @@ class TudouIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         youku_vcode = self._search_regex(
-            r'vcode:\s*[\'"](.+?)[\'"]', webpage, 'youku vcode', default=None)
+            r'vcode\s*:\s*[\'"]([^\'"]*)[\'"]', webpage, 'youku vcode', default=None)
         if youku_vcode:
             return self.url_result('youku:' + youku_vcode, ie='Youku')
 
         title = self._search_regex(
-            r",kw:\s*['\"](.+?)[\"']", webpage, 'title')
+            r',kw\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'title')
         thumbnail_url = self._search_regex(
-            r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False)
+            r',pic\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'thumbnail URL', fatal=False)
 
         player_url = self._search_regex(
-            r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
+            r'playerUrl\s*:\s*[\'"]([^\'"]+\.swf)[\'"]',
             webpage, 'player URL', default=self._PLAYER_URL)
 
         segments = self._parse_json(self._search_regex(
-            r'segs: \'(.*)\'', webpage, 'segments'), video_id)
+            r'segs: \'([^\']+)\'', webpage, 'segments'), video_id)
         # It looks like the keys are the arguments that have to be passed as
         # the hd field in the request url, we pick the higher
         # Also, filter non-number qualities (see issue #3643).

From b264c2130221912adfc7cc35d73c2a88d79eafeb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 02:57:14 +0800
Subject: [PATCH 1307/2145] [tudou] Use single quotes and compat_str

---
 youtube_dl/extractor/tudou.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 53ba8511f..5f7ac4b35 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 
 
 class TudouIE(InfoExtractor):
@@ -32,7 +33,7 @@ class TudouIE(InfoExtractor):
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
 
     def _url_for_id(self, video_id, quality=None):
-        info_url = "http://v2.tudou.com/f?id=" + str(video_id)
+        info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
         if quality:
             info_url += '&hd' + quality
         xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")

From 2ffe3bc14b5e65c902fe5ddd610143c791edaa52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 04:15:49 +0600
Subject: [PATCH 1308/2145] [ndr] Rework and cover with tests

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/ndr.py      | 469 ++++++++++++++++++++++---------
 2 files changed, 334 insertions(+), 136 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 44ab7ce3c..fadba905d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -367,6 +367,7 @@ from .nbc import (
 from .ndr import (
     NDRIE,
     NJoyIE,
+    NDREmbedBaseIE,
     NDREmbedIE,
     NJoyEmbedIE,
 )
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 87f3edbbe..e3cc6fde8 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -1,183 +1,380 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     int_or_none,
+    parse_iso8601,
     qualities,
 )
 
 
-preference = qualities(['xs', 's', 'm','l', 'xl'])
-
-
 class NDRBaseIE(InfoExtractor):
-
-    def extract_video_info(self, playlist, video_id):
-        formats = []
-        streamType = playlist.get('config').get('streamType')
-        if streamType == 'httpVideo':
-            for key, f in playlist.items():
-                if key != 'config':
-                    src = f['src']
-                    if '.f4m' in src:
-                        formats.extend(self._extract_f4m_formats(src, video_id))
-                    elif '.m3u8' in src:
-                        formats.extend(self._extract_m3u8_formats(src, video_id, fatal=False))
-                    else:
-                        quality = f.get('quality')
-                        formats.append({
-                            'url': src,
-                            'format_id': quality,
-                            'preference': preference(quality),
-                        })
-        elif streamType == 'httpAudio':
-            for key, f in playlist.items():
-                if key != 'config':
-                    formats.append({
-                        'url': f['src'],
-                        'format_id': 'mp3',
-                        'vcodec': 'none',
-                    })
-        else:
-            raise ExtractorError('No media links available for %s' % video_id)
-
-        self._sort_formats(formats)
-
-        config = playlist.get('config')
-
-        title = config['title']
-        duration = int_or_none(config.get('duration'))
-        thumbnails = [{
-            'id': thumbnail.get('quality'),
-            'url': thumbnail.get('src'),
-            'preference': preference(thumbnail.get('quality'))
-        } for thumbnail in config.get('poster').values()]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnails': thumbnails,
-            'duration': duration,
-            'formats': formats,
-        }
-
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
-
-        if not json_data:
-            webpage = self._download_webpage(url, video_id)
-            embed_url = self._html_search_regex(r'<iframe[^>]+id="pp_\w+"[^>]+src="(/.*)"', webpage, 'embed url', None, False)
-            if not embed_url:
-                embed_url = self._html_search_meta('embedURL', webpage, fatal=False)
-            if embed_url:
-                if embed_url.startswith('/'):
-                    return self.url_result('http://www.ndr.de%s' % embed_url, 'NDREmbed')
-                else:
-                    return self.url_result(embed_url, 'NDREmbed')
-            raise ExtractorError('No media links available for %s' % video_id)
-
-        return self.extract_video_info(json_data['playlist'], video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        return self._extract_embed(webpage, display_id)
 
 
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
-    IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?,(?P<id>\w+)\.html'
+    IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _TESTS = [{
+        # httpVideo, same content id
+        'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
+        'md5': '6515bc255dc5c5f8c85bbc38e035a659',
+        'info_dict': {
+            'id': 'hafengeburtstag988',
+            'display_id': 'Party-Poette-und-Parade',
+            'ext': 'mp4',
+            'title': 'Party, Pötte und Parade',
+            'description': 'md5:ad14f9d2f91d3040b6930c697e5f6b4c',
+            'uploader': 'ndrtv',
+            'timestamp': 1431108900,
+            'upload_date': '20150510',
+            'duration': 3498,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpVideo, different content id
+        'url': 'http://www.ndr.de/sport/fussball/40-Osnabrueck-spielt-sich-in-einen-Rausch,osna270.html',
+        'md5': '1043ff203eab307f0c51702ec49e9a71',
+        'info_dict': {
+            'id': 'osna272',
+            'display_id': '40-Osnabrueck-spielt-sich-in-einen-Rausch',
+            'ext': 'mp4',
+            'title': 'Osnabrück - Wehen Wiesbaden: Die Highlights',
+            'description': 'md5:32e9b800b3d2d4008103752682d5dc01',
+            'uploader': 'ndrtv',
+            'timestamp': 1442059200,
+            'upload_date': '20150912',
+            'duration': 510,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpAudio, same content id
+        'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
+        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+        'info_dict': {
+            'id': 'audio51535',
+            'display_id': 'La-Valette-entgeht-der-Hinrichtung',
+            'ext': 'mp3',
+            'title': 'La Valette entgeht der Hinrichtung',
+            'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
+            'uploader': 'ndrinfo',
+            'timestamp': 1290626100,
+            'upload_date': '20140729',
+            'duration': 884,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
-    _TESTS = [
-        {
-            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
-            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
-            'note': 'Video file',
-            'info_dict': {
-                'id': 'nordmagazin25866',
-                'ext': 'mp4',
-                'title': 'Kartoffeltage in der Lewitz',
-                'duration': 166,
-            },
-            'skip': '404 Not found',
-        },
-        {
-            'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
-            'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
-            'info_dict': {
-                'id': 'hafengeburtstag988',
-                'ext': 'mp4',
-                'title': 'Party, Pötte und Parade',
-                'duration': 3498,
-            },
-        },
-        {
-            'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
-            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
-            'note': 'Audio file',
-            'info_dict': {
-                'id': 'audio51535',
-                'ext': 'mp3',
-                'title': 'La Valette entgeht der Hinrichtung',
-                'duration': 884,
-            }
+    def _extract_embed(self, webpage, display_id):
+        embed_url = self._html_search_meta(
+            'embedURL', webpage, 'embed URL', fatal=True)
+        description = self._search_regex(
+            r'<p[^>]+itemprop="description">([^<]+)</p>',
+            webpage, 'description', fatal=False)
+        timestamp = parse_iso8601(
+            self._search_regex(
+                r'<span itemprop="datePublished" content="([^"]+)">',
+                webpage, 'upload date', fatal=False))
+        return {
+            '_type': 'url_transparent',
+            'url': embed_url,
+            'display_id': display_id,
+            'description': description,
+            'timestamp': timestamp,
         }
-    ]
 
 
 class NJoyIE(NDRBaseIE):
-    IE_NAME = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/.+?,(?P<id>\w+)\.html'
-
-    _TEST = {
+    IE_NAME = 'njoy'
+    IE_DESC = 'N-JOY'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _TESTS = [{
+        # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
         'md5': 'cb63be60cd6f9dd75218803146d8dc67',
         'info_dict': {
             'id': 'comedycontest2480',
+            'display_id': 'Benaissa-beim-NDR-Comedy-Contest',
             'ext': 'mp4',
             'title': 'Benaissa beim NDR Comedy Contest',
+            'description': 'md5:f057a6c4e1c728b10d33b5ffd36ddc39',
+            'uploader': 'ndrtv',
+            'upload_date': '20141129',
             'duration': 654,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpVideo, different content id
+        'url': 'http://www.n-joy.de/musik/Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-,felixjaehn168.html',
+        'md5': '417660fffa90e6df2fda19f1b40a64d8',
+        'info_dict': {
+            'id': 'dockville882',
+            'display_id': 'Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-',
+            'ext': 'mp4',
+            'title': '"Ich hab noch nie" mit Felix Jaehn',
+            'description': 'md5:85dd312d53be1b99e1f998a16452a2f3',
+            'uploader': 'njoy',
+            'upload_date': '20150822',
+            'duration': 211,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _extract_embed(self, webpage, display_id):
+        video_id = self._search_regex(
+            r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
+        description = self._search_regex(
+            r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
+            webpage, 'description', fatal=False)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'NDREmbedBase',
+            'url': 'ndr:%s' % video_id,
+            'display_id': display_id,
+            'description': description,
         }
-    }
 
 
-class NDREmbedBaseIE(NDRBaseIE):
+class NDREmbedBaseIE(InfoExtractor):
+    IE_NAME = 'ndr:embed:base'
+    _VALID_URL = r'(?:ndr:(?P<id_s>[\da-z]+)|https?://www\.ndr\.de/(?P<id>[\da-z]+)-ppjson\.json)'
+    _TESTS = [{
+        'url': 'ndr:soundcheck3366',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/soundcheck3366-ppjson.json',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
-        if not json_data:
-            raise ExtractorError('No media links available for %s' % video_id)
-        return self.extract_video_info(json_data['playlist'], video_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('id_s')
+
+        ppjson = self._download_json(
+            'http://www.ndr.de/%s-ppjson.json' % video_id, video_id)
+
+        playlist = ppjson['playlist']
+
+        formats = []
+        quality_key = qualities(('xs', 's', 'm', 'l', 'xl'))
+
+        for format_id, f in playlist.items():
+            src = f.get('src')
+            if not src:
+                continue
+            ext = determine_ext(src, None)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, m3u8_id='hls', entry_protocol='m3u8_native'))
+            else:
+                quality = f.get('quality')
+                ff = {
+                    'url': src,
+                    'format_id': quality or format_id,
+                    'quality': quality_key(quality),
+                }
+                type_ = f.get('type')
+                if type_ and type_.split('/')[0] == 'audio':
+                    ff['vcodec'] = 'none'
+                    ff['ext'] = ext or 'mp3'
+                formats.append(ff)
+        self._sort_formats(formats)
+
+        config = playlist['config']
+
+        live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive']
+        title = config['title']
+        if live:
+            title = self._live_title(title)
+        uploader = ppjson.get('config', {}).get('branding')
+        upload_date = ppjson.get('config', {}).get('publicationDate')
+        duration = int_or_none(config.get('duration'))
+
+        thumbnails = [{
+            'id': thumbnail.get('quality') or thumbnail_id,
+            'url': thumbnail['src'],
+            'preference': quality_key(thumbnail.get('quality')),
+        } for thumbnail_id, thumbnail in config.get('poster', {}).items() if thumbnail.get('src')]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'is_live': live,
+            'uploader': uploader if uploader != '-' else None,
+            'upload_date': upload_date[0:8] if upload_date else None,
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
 
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
-
-    _TEST = {
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
-        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
         'info_dict': {
             'id': 'ndraktuell28488',
             'ext': 'mp4',
             'title': 'Norddeutschland begrüßt Flüchtlinge',
+            'is_live': False,
+            'uploader': 'ndrtv',
+            'upload_date': '20150907',
             'duration': 132,
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html',
+        'md5': '002085c44bae38802d94ae5802a36e78',
+        'info_dict': {
+            'id': 'soundcheck3366',
+            'ext': 'mp4',
+            'title': 'Ella Henderson braucht Vergleiche nicht zu scheuen',
+            'is_live': False,
+            'uploader': 'ndr2',
+            'upload_date': '20150912',
+            'duration': 3554,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.ndr.de/info/audio51535-player.html',
+        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+        'info_dict': {
+            'id': 'audio51535',
+            'ext': 'mp3',
+            'title': 'La Valette entgeht der Hinrichtung',
+            'is_live': False,
+            'uploader': 'ndrinfo',
+            'upload_date': '20140729',
+            'duration': 884,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/visite/visite11010-externalPlayer.html',
+        'md5': 'ae57f80511c1e1f2fd0d0d3d31aeae7c',
+        'info_dict': {
+            'id': 'visite11010',
+            'ext': 'mp4',
+            'title': 'Visite - die ganze Sendung',
+            'is_live': False,
+            'uploader': 'ndrtv',
+            'upload_date': '20150902',
+            'duration': 3525,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpVideoLive
+        'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html',
+        'info_dict': {
+            'id': 'livestream217',
+            'ext': 'flv',
+            'title': 're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+            'upload_date': '20150910',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.ndr.de/ndrkultur/audio255020-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/nordtour/nordtour7124-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/kultur/film/videos/videoimport10424-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/hamburg_journal/hamj43006-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/weltbilder/weltbilder4518-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/doku952-player.html',
+        'only_matching': True,
+    }]
 
 
 class NJoyEmbedIE(NDREmbedBaseIE):
-    IE_NAME = 'N-JOY:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
-
-    _TEST = {
-        'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html',
-        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+    IE_NAME = 'njoy:embed'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+    _TESTS = [{
+        # httpVideo
+        'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
+        'md5': '8483cbfe2320bd4d28a349d62d88bd74',
         'info_dict': {
-            'id': 'portraet374',
+            'id': 'doku948',
             'ext': 'mp4',
-            'title': 'Viviane Andereggen - "Schuld um Schuld"',
-            'duration': 129,
-        }
-    }
+            'title': 'Zehn Jahre Reeperbahn Festival - die Doku',
+            'is_live': False,
+            'upload_date': '20150807',
+            'duration': 1011,
+        },
+    }, {
+        # httpAudio
+        'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html',
+        'md5': 'd989f80f28ac954430f7b8a48197188a',
+        'info_dict': {
+            'id': 'stefanrichter100',
+            'ext': 'mp3',
+            'title': 'Interview mit einem Augenzeugen',
+            'is_live': False,
+            'uploader': 'njoy',
+            'upload_date': '20150909',
+            'duration': 140,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpAudioLive, no explicit ext
+        'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html',
+        'info_dict': {
+            'id': 'webradioweltweit100',
+            'ext': 'mp3',
+            'title': 're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+            'uploader': 'njoy',
+            'upload_date': '20150810',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.n-joy.de/musik/dockville882-player_image-3905259e-0803-4764-ac72-8b7de077d80a_theme-n-joy.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.n-joy.de/radio/sendungen/morningshow/urlaubsfotos190-player_image-066a5df1-5c95-49ec-a323-941d848718db_theme-n-joy.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.n-joy.de/entertainment/comedy/krudetv290-player_image-ab261bfe-51bf-4bf3-87ba-c5122ee35b3d_theme-n-joy.html',
+        'only_matching': True,
+    }]

From 82c18e2a53a14cf88bc15a1d0b0f4342d2dc5db8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:18:57 +0600
Subject: [PATCH 1309/2145] [nowness] Simplify

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowness.py  | 107 ++++++++++++++++---------------
 2 files changed, 57 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 99da52d96..74b7df463 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -408,7 +408,7 @@ from .novamov import NovaMovIE
 from .nowness import (
     NownessIE,
     NownessPlaylistIE,
-    NownessSerieIE,
+    NownessSeriesIE,
 )
 from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 9a0e46917..9fb916c18 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -4,11 +4,14 @@ from __future__ import unicode_literals
 from .brightcove import BrightcoveIE
 from .common import InfoExtractor
 from ..utils import ExtractorError
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_str,
+    compat_urllib_request,
+)
 
 
 class NownessBaseIE(InfoExtractor):
-    def extract_url_result(self, post):
+    def _extract_url_result(self, post):
         if post['type'] == 'video':
             for media in post['media']:
                 if media['type'] == 'video':
@@ -18,7 +21,7 @@ class NownessBaseIE(InfoExtractor):
                         player_code = self._download_webpage(
                             'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
                             note='Downloading player JavaScript',
-                            errnote='Player download failed')
+                            errnote='Unable to download player JavaScript')
                         bc_url = BrightcoveIE._extract_brightcove_url(player_code)
                         if bc_url is None:
                             raise ExtractorError('Could not find player definition')
@@ -32,50 +35,46 @@ class NownessBaseIE(InfoExtractor):
                         # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
                         pass
 
-    def api_request(self, url, request_path):
+    def _api_request(self, url, request_path):
         display_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/' + request_path % display_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        json_data = self._download_json(request, display_id)
-        return display_id, json_data
+        request = compat_urllib_request.Request(
+            'http://api.nowness.com/api/' + request_path % display_id,
+            headers={
+                'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
+            })
+        return display_id, self._download_json(request, display_id)
 
 
 class NownessIE(NownessBaseIE):
     IE_NAME = 'nowness'
     _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P<id>[^/]+?)(?:$|[?#])'
-    _TESTS = [
-        {
-            'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
-            'md5': '068bc0202558c2e391924cb8cc470676',
-            'info_dict': {
-                'id': '2520295746001',
-                'ext': 'mp4',
-                'title': 'Candor: The Art of Gesticulation',
-                'description': 'Candor: The Art of Gesticulation',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'uploader': 'Nowness',
-            }
-        },
-        {
-            'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
-            'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
-            'info_dict': {
-                'id': '3716354522001',
-                'ext': 'mp4',
-                'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
-                'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'uploader': 'Nowness',
-            }
-        },
-    ]
+    _TESTS = [{
+        'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
+        'md5': '068bc0202558c2e391924cb8cc470676',
+        'info_dict': {
+            'id': '2520295746001',
+            'ext': 'mp4',
+            'title': 'Candor: The Art of Gesticulation',
+            'description': 'Candor: The Art of Gesticulation',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'Nowness',
+        }
+    }, {
+        'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
+        'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
+        'info_dict': {
+            'id': '3716354522001',
+            'ext': 'mp4',
+            'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
+            'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'Nowness',
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id, post = self.api_request(url, 'post/getBySlug/%s')
-        return self.extract_url_result(post)
+        _, post = self._api_request(url, 'post/getBySlug/%s')
+        return self._extract_url_result(post)
 
 
 class NownessPlaylistIE(NownessBaseIE):
@@ -83,33 +82,39 @@ class NownessPlaylistIE(NownessBaseIE):
     _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
-        'info_dict':
-        {
+        'info_dict': {
             'id': '3286',
         },
         'playlist_mincount': 8,
     }
 
     def _real_extract(self, url):
-        playlist_id, playlist = self.api_request(url, 'post?PlaylistId=%s')
-        entries = [self.extract_url_result(item) for item in playlist['items']]
+        playlist_id, playlist = self._api_request(url, 'post?PlaylistId=%s')
+        entries = [self._extract_url_result(item) for item in playlist['items']]
         return self.playlist_result(entries, playlist_id)
 
 
-class NownessSerieIE(NownessBaseIE):
-    IE_NAME = 'nowness:serie'
+class NownessSeriesIE(NownessBaseIE):
+    IE_NAME = 'nowness:series'
     _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[^/]+?)(?:$|[?#])'
     _TEST = {
         'url': 'https://www.nowness.com/series/60-seconds',
-        'info_dict':
-        {
+        'info_dict': {
             'id': '60',
+            'title': '60 Seconds',
+            'description': 'One-minute wisdom in a new NOWNESS series',
         },
         'playlist_mincount': 4,
     }
 
     def _real_extract(self, url):
-        display_id, serie = self.api_request(url, 'series/getBySlug/%s')
-        serie_id = str(serie['id'])
-        entries = [self.extract_url_result(post) for post in serie['posts']]
-        return self.playlist_result(entries, serie_id)
+        display_id, series = self._api_request(url, 'series/getBySlug/%s')
+        entries = [self._extract_url_result(post) for post in series['posts']]
+        series_title = None
+        series_description = None
+        translations = series.get('translations', [])
+        if translations:
+            series_title = translations[0].get('title') or translations[0]['seoTitle']
+            series_description = translations[0].get('seoDescription')
+        return self.playlist_result(
+            entries, compat_str(series['id']), series_title, series_description)

From c246773599f6b541033ac926ac6ed360fb63f6ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:27:22 +0600
Subject: [PATCH 1310/2145] [nowness] Add vimeo test

---
 youtube_dl/extractor/nowness.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 9fb916c18..b97f62fdb 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -58,7 +58,7 @@ class NownessIE(NownessBaseIE):
             'description': 'Candor: The Art of Gesticulation',
             'thumbnail': 're:^https?://.*\.jpg',
             'uploader': 'Nowness',
-        }
+        },
     }, {
         'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
         'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
@@ -69,7 +69,21 @@ class NownessIE(NownessBaseIE):
             'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
             'thumbnail': 're:^https?://.*\.jpg',
             'uploader': 'Nowness',
-        }
+        },
+    }, {
+        # vimeo
+        'url': 'https://www.nowness.com/series/nowness-picks/jean-luc-godard-supercut',
+        'md5': '9a5a6a8edf806407e411296ab6bc2a49',
+        'info_dict': {
+            'id': '130020913',
+            'ext': 'mp4',
+            'title': 'Bleu, Blanc, Rouge - A Godard Supercut',
+            'description': 'md5:f0ea5f1857dffca02dbd37875d742cec',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150607',
+            'uploader': 'Cinema Sem Lei',
+            'uploader_id': 'cinemasemlei',
+        },
     }]
 
     def _real_extract(self, url):

From ac7a1b0dfb9e1e3ea384dd36a425ed9483ee17ef Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:31:20 +0600
Subject: [PATCH 1311/2145] [README.md] Clarify playlist_index description

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 24bfe38a2..27b20df04 100644
--- a/README.md
+++ b/README.md
@@ -278,7 +278,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
  - `playlist`: The name or the id of the playlist that contains the video.
- - `playlist_index`: The index of the video in the playlist, a five-digit number.
+ - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.

From 31208a07c247f5b8f1f0fc26da1d477f7440e513 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:34:09 +0600
Subject: [PATCH 1312/2145] [README.md] Unify playlist description

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 27b20df04..1a00a02bf 100644
--- a/README.md
+++ b/README.md
@@ -277,7 +277,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
- - `playlist`: The name or the id of the playlist that contains the video.
+ - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video.
  - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
 

From e2ff3df314c7e710d429ca311b2a22839d4aa3f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Sep 2015 03:35:15 +0600
Subject: [PATCH 1313/2145] [clubic] Relax _VALID_URL (Closes #6854)

---
 youtube_dl/extractor/clubic.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clubic.py b/youtube_dl/extractor/clubic.py
index 14f215c5c..1dfa7c12e 100644
--- a/youtube_dl/extractor/clubic.py
+++ b/youtube_dl/extractor/clubic.py
@@ -12,9 +12,9 @@ from ..utils import (
 
 
 class ClubicIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?clubic\.com/video/[^/]+/video.*-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'http://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html',
         'md5': '1592b694ba586036efac1776b0b43cd3',
         'info_dict': {
@@ -24,7 +24,10 @@ class ClubicIE(InfoExtractor):
             'description': 're:Gueule de bois chez Nokia. Le constructeur a indiqué cette.*',
             'thumbnail': 're:^http://img\.clubic\.com/.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.clubic.com/video/video-clubic-week-2-0-apple-iphone-6s-et-plus-mais-surtout-le-pencil-469792.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 953fed280f1a35bc6e8a701a25b8450ec08efdd3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Sep 2015 00:25:08 +0200
Subject: [PATCH 1314/2145] [compat] Do not use unicode

If the code ever runs on 3.x, it would fail. Even if it never does, the unicode name confuses Python 3 code analysis tools.
---
 youtube_dl/compat.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e32bef279..1ff42d94b 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -80,6 +80,11 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+try:
+    compat_str = unicode  # Python 2
+except NameError:
+    compat_str = str
+
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
@@ -100,7 +105,7 @@ except ImportError:  # Python 2
             # Is it a string-like object?
             string.split
             return b''
-        if isinstance(string, unicode):
+        if isinstance(string, compat_str):
             string = string.encode('utf-8')
         bits = string.split(b'%')
         if len(bits) == 1:
@@ -150,11 +155,6 @@ except ImportError:  # Python 2
         string = string.replace('+', ' ')
         return compat_urllib_parse_unquote(string, encoding, errors)
 
-try:
-    compat_str = unicode  # Python 2
-except NameError:
-    compat_str = str
-
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
@@ -234,7 +234,7 @@ else:
     # Working around shlex issue with unicode strings on some python 2
     # versions (see http://bugs.python.org/issue1548891)
     def compat_shlex_split(s, comments=False, posix=True):
-        if isinstance(s, unicode):
+        if isinstance(s, compat_str):
             s = s.encode('utf-8')
         return shlex.split(s, comments, posix)
 

From 287be8c615574f90f4fac5adad0b32b3102f621e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Sep 2015 00:26:12 +0200
Subject: [PATCH 1315/2145] [youtube:truncated_url] Add ?t=

Sometimes found in links - ?t=123&v=ABCD starts the video at position 123.
---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 97ce36550..abacdfaed 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1970,6 +1970,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             annotation_id=annotation_[^&]+|
             x-yt-cl=[0-9]+|
             hl=[^&]*|
+            t=[0-9]+
         )?
         |
             attribution_link\?a=[^&]+
@@ -1992,6 +1993,9 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'https://www.youtube.com/watch?hl=en-GB',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?t=2372',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 2b3c2546782f84e0b3fe42b55d2030e5350cbe92 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Sep 2015 00:32:20 +0200
Subject: [PATCH 1316/2145] [youtube:channel] Correct 404 handling

Previously, when we encountered a 404 - such as youtube-dl https://www.youtube.com/ohJeiboh8oorehai - we crashed with a regexp error. Instead, make sure to go on and eventually report a 404.
---
 youtube_dl/extractor/youtube.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index abacdfaed..b252e36e1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1654,12 +1654,15 @@ class YoutubeChannelIE(InfoExtractor):
         channel_page = self._download_webpage(
             url + '?view=57', channel_id,
             'Downloading channel page', fatal=False)
-        channel_playlist_id = self._html_search_meta(
-            'channelId', channel_page, 'channel id', default=None)
-        if not channel_playlist_id:
-            channel_playlist_id = self._search_regex(
-                r'data-channel-external-id="([^"]+)"',
-                channel_page, 'channel id', default=None)
+        if channel_page is False:
+            channel_playlist_id = False
+        else:
+            channel_playlist_id = self._html_search_meta(
+                'channelId', channel_page, 'channel id', default=None)
+            if not channel_playlist_id:
+                channel_playlist_id = self._search_regex(
+                    r'data-channel-external-id="([^"]+)"',
+                    channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
             return self.url_result(

From 3b9264a04902c4ce01d3fac67bf51799fece0e53 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 14 Sep 2015 23:07:03 +0600
Subject: [PATCH 1317/2145] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1a00a02bf..42656aae1 100644
--- a/README.md
+++ b/README.md
@@ -261,7 +261,7 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or to place it in [configuration file](#configuration).
+To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or place it in [configuration file](#configuration).
 
 On Windows you may also need to setup `%HOME%` environment variable manually.
 

From 0391bc8176c22e586f1c866145bc31e9cd2c50ed Mon Sep 17 00:00:00 2001
From: Alex Vong <alexvong1995@gmail.com>
Date: Mon, 10 Aug 2015 12:55:29 +0800
Subject: [PATCH 1318/2145] Add the missing tag `EMBEDDING YOUTUBE-DL` in the
 content table.

* README.md: Add missing tag.
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 42656aae1..2ed751791 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@ youtube-dl - download videos from youtube.com or other video platforms
 - [VIDEO SELECTION](#video-selection)
 - [FAQ](#faq)
 - [DEVELOPER INSTRUCTIONS](#developer-instructions)
+- [EMBEDDING YOUTUBE-DL](#embedding-youtube-dl)
 - [BUGS](#bugs)
 - [COPYRIGHT](#copyright)
 

From 272e4db5c746ef4c94b7c9e7bac0fc008319b158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Sep 2015 21:24:01 +0600
Subject: [PATCH 1319/2145] [pornhub] Relax _VALID_URL (Closes #6868)

---
 youtube_dl/extractor/pornhub.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 7b0cdc41a..a656ad85a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -20,7 +20,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
+    _VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
@@ -34,6 +34,9 @@ class PornHubIE(InfoExtractor):
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
         'only_matching': True,
+    }, {
+        'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
+        'only_matching': True,
     }]
 
     @classmethod

From 88060cce108f7436f8747186855ef0d63fa23338 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Sep 2015 21:57:30 +0600
Subject: [PATCH 1320/2145] [rai] Add support for videos embedded with
 drawMediaRaiTV (Closes #6866)

---
 youtube_dl/extractor/rai.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 1631faf29..7ff1d06c4 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     parse_duration,
@@ -72,6 +73,18 @@ class RaiIE(InfoExtractor):
                 'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!',
                 'uploader': 'RaiTre',
             }
+        },
+        {
+            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
+            'md5': '037104d2c14132887e5e4cf114569214',
+            'info_dict': {
+                'id': '0c7a664b-d0f4-4b2c-8835-3f82e46f433e',
+                'ext': 'flv',
+                'title': 'Il pacco',
+                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+                'uploader': 'RaiTre',
+                'upload_date': '20141221',
+            },
         }
     ]
 
@@ -90,11 +103,14 @@ class RaiIE(InfoExtractor):
         relinker_url = self._extract_relinker_url(webpage)
 
         if not relinker_url:
-            iframe_path = self._search_regex(
-                r'<iframe[^>]+src="/?(dl/[^"]+\?iframe\b[^"]*)"',
+            iframe_url = self._search_regex(
+                [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
+                 r'drawMediaRaiTV\(["\'](.+?)["\']'],
                 webpage, 'iframe')
+            if not iframe_url.startswith('http'):
+                iframe_url = compat_urlparse.urljoin(url, iframe_url)
             webpage = self._download_webpage(
-                '%s/%s' % (host, iframe_path), video_id)
+                iframe_url, video_id)
             relinker_url = self._extract_relinker_url(webpage)
 
         relinker = self._download_json(

From 12bc2429444b9f89d1b7f4f631c25cc3af2a1d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:25:25 +0600
Subject: [PATCH 1321/2145] [telecinco] Incorporate mitele code

---
 youtube_dl/extractor/telecinco.py | 54 +++++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index ae94f055c..c0ad286c5 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -1,10 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .mitele import MiTeleIE
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
+from ..utils import (
+    get_element_by_attribute,
+    parse_duration,
+    strip_jsonp,
+)
 
 
-class TelecincoIE(MiTeleIE):
+class TelecincoIE(InfoExtractor):
     IE_NAME = 'telecinco.es'
     _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P<id>.+?)\.html'
 
@@ -27,3 +39,41 @@ class TelecincoIE(MiTeleIE):
         'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
         'only_matching': True,
     }]
+
+    def _real_extract(self, url):
+        episode = self._match_id(url)
+        webpage = self._download_webpage(url, episode)
+        embed_data_json = self._search_regex(
+            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
+        ).replace('\'', '"')
+        embed_data = json.loads(embed_data_json)
+
+        domain = embed_data['mediaUrl']
+        if not domain.startswith('http'):
+            # only happens in telecinco.es videos
+            domain = 'http://' + domain
+        info_url = compat_urlparse.urljoin(
+            domain,
+            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
+        )
+        info_el = self._download_xml(info_url, episode).find('./video/info')
+
+        video_link = info_el.find('videoUrl/link').text
+        token_query = compat_urllib_parse.urlencode({'id': video_link})
+        token_info = self._download_json(
+            embed_data['flashvars']['ov_tk'] + '?' + token_query,
+            episode,
+            transform_source=strip_jsonp
+        )
+        formats = self._extract_m3u8_formats(
+            token_info['tokenizedUrl'], episode, ext='mp4')
+
+        return {
+            'id': embed_data['videoId'],
+            'display_id': episode,
+            'title': info_el.find('title').text,
+            'formats': formats,
+            'description': get_element_by_attribute('class', 'text', webpage),
+            'thumbnail': info_el.find('thumb').text,
+            'duration': parse_duration(info_el.find('duration').text),
+        }

From f84ce1ebaf930f5fc422e4fab77fa9c7c86a999a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:27:03 +0600
Subject: [PATCH 1322/2145] [mitele] Fix extraction (Closes #6414)

---
 youtube_dl/extractor/mitele.py | 105 ++++++++++++++++++---------------
 1 file changed, 58 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 852d72266..9e8a8fc72 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,17 +1,11 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_parse_unquote,
-    compat_urlparse,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
+    encode_dict,
     get_element_by_attribute,
-    parse_duration,
-    strip_jsonp,
+    int_or_none,
 )
 
 
@@ -21,54 +15,71 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
+        'md5': 'ace7635b2a0b286aaa37d3ff192d2a8a',
         'info_dict': {
-            'id': '0fce117d',
-            'ext': 'mp4',
-            'title': 'Programa 144 - Tor, la web invisible',
-            'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
+            'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',
+            'ext': 'flv',
+            'title': 'Tor, la web invisible',
+            'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
+            'thumbnail': 're:(?i)^https?://.*\.jpg$',
             'duration': 2913,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }]
 
     def _real_extract(self, url):
-        episode = self._match_id(url)
-        webpage = self._download_webpage(url, episode)
-        embed_data_json = self._search_regex(
-            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
-        ).replace('\'', '"')
-        embed_data = json.loads(embed_data_json)
+        display_id = self._match_id(url)
 
-        domain = embed_data['mediaUrl']
-        if not domain.startswith('http'):
-            # only happens in telecinco.es videos
-            domain = 'http://' + domain
-        info_url = compat_urlparse.urljoin(
-            domain,
-            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
-        )
-        info_el = self._download_xml(info_url, episode).find('./video/info')
+        webpage = self._download_webpage(url, display_id)
 
-        video_link = info_el.find('videoUrl/link').text
-        token_query = compat_urllib_parse.urlencode({'id': video_link})
-        token_info = self._download_json(
-            embed_data['flashvars']['ov_tk'] + '?' + token_query,
-            episode,
-            transform_source=strip_jsonp
-        )
-        formats = self._extract_m3u8_formats(
-            token_info['tokenizedUrl'], episode, ext='mp4')
+        config_url = self._search_regex(
+            r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url')
+
+        config = self._download_json(
+            config_url, display_id, 'Downloading config JSON')
+
+        mmc = self._download_json(
+            config['services']['mmc'], display_id, 'Downloading mmc JSON')
+
+        formats = []
+        for location in mmc['locations']:
+            gat = self._proto_relative_url(location.get('gat'), 'http:')
+            bas = location.get('bas')
+            loc = location.get('loc')
+            ogn = location.get('ogn')
+            if None in (gat, bas, loc, ogn):
+                continue
+            token_data = {
+                'bas': bas,
+                'icd': loc,
+                'ogn': ogn,
+                'sta': '0',
+            }
+            media = self._download_json(
+                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data)).encode('utf-8')),
+                display_id, 'Downloading %s JSON' % location['loc'])
+            file_ = media.get('file')
+            if not file_:
+                continue
+            formats.extend(self._extract_f4m_formats(
+                file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                display_id, f4m_id=loc))
+
+        title = self._search_regex(
+            r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>', webpage, 'title')
+
+        video_id = self._search_regex(
+            r'data-media-id\s*=\s*"([^"]+)"', webpage,
+            'data media id', default=None) or display_id
+        thumbnail = config.get('poster', {}).get('imageUrl')
+        duration = int_or_none(mmc.get('duration'))
 
         return {
-            'id': embed_data['videoId'],
-            'display_id': episode,
-            'title': info_el.find('title').text,
-            'formats': formats,
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
             'description': get_element_by_attribute('class', 'text', webpage),
-            'thumbnail': info_el.find('thumb').text,
-            'duration': parse_duration(info_el.find('duration').text),
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From d5e7657fe2c045545e3cdc089413661befad8b4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:37:01 +0600
Subject: [PATCH 1323/2145] [telecinco] Add support or cuatro.com

---
 youtube_dl/extractor/telecinco.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index c0ad286c5..018fd7d34 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 class TelecincoIE(InfoExtractor):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
@@ -32,6 +32,15 @@ class TelecincoIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
+        'md5': '0a5b9f3cc8b074f50a0578f823a12694',
+        'info_dict': {
+            'id': 'MDSVID20150916_0128',
+            'ext': 'mp4',
+            'title': '¿Quién es este ex futbolista con el que hablan ...',
+            'duration': 79,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,
@@ -66,7 +75,7 @@ class TelecincoIE(InfoExtractor):
             transform_source=strip_jsonp
         )
         formats = self._extract_m3u8_formats(
-            token_info['tokenizedUrl'], episode, ext='mp4')
+            token_info['tokenizedUrl'], episode, ext='mp4', entry_protocol='m3u8_native')
 
         return {
             'id': embed_data['videoId'],

From 369e60162e9cc99c89dac52a40297b68cbc82392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:37:47 +0600
Subject: [PATCH 1324/2145] [telecinco] Update test

---
 youtube_dl/extractor/telecinco.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 018fd7d34..685f1db40 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -22,16 +22,13 @@ class TelecincoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
+        'md5': '5cbef3ad5ef17bf0d21570332d140729',
         'info_dict': {
             'id': 'MDSVID20141015_0058',
             'ext': 'mp4',
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }, {
         'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
         'md5': '0a5b9f3cc8b074f50a0578f823a12694',

From 0e1b2566fff795981d0f7cccbca6700a3fc260a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:39:24 +0600
Subject: [PATCH 1325/2145] [telecinco] Clarify IE_DESC

---
 youtube_dl/extractor/telecinco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 685f1db40..9740c3452 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class TelecincoIE(InfoExtractor):
-    IE_NAME = 'telecinco.es'
+    IE_DESC = 'telecinco.es and cuatro.es'
     _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{

From 3368d70dceba93aa9acdb97ec2705471a3424c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:39:48 +0600
Subject: [PATCH 1326/2145] [mitele] Clarify IE_DESC

---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 9e8a8fc72..54993e2c9 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MiTeleIE(InfoExtractor):
-    IE_NAME = 'mitele.es'
+    IE_DESC = 'mitele.es'
     _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
     _TESTS = [{

From d492dad8f440c8d0d006a896de0a56b01cd46496 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:45:39 +0600
Subject: [PATCH 1327/2145] [telecinco] Add support for mediaset.es

---
 youtube_dl/extractor/telecinco.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 9740c3452..b0bf9ab8d 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -17,8 +17,8 @@ from ..utils import (
 
 
 class TelecincoIE(InfoExtractor):
-    IE_DESC = 'telecinco.es and cuatro.es'
-    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P<id>.+?)\.html'
+    IE_DESC = 'telecinco.es, cuatro.es and mediaset.es'
+    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
@@ -38,6 +38,15 @@ class TelecincoIE(InfoExtractor):
             'title': '¿Quién es este ex futbolista con el que hablan ...',
             'duration': 79,
         },
+    }, {
+        'url': 'http://www.mediaset.es/12meses/campanas/doylacara/conlatratanohaytrato/Ayudame-dar-cara-trata-trato_2_1986630220.html',
+        'md5': 'ad1bfaaba922dd4a295724b05b68f86a',
+        'info_dict': {
+            'id': 'MDSVID20150513_0220',
+            'ext': 'mp4',
+            'title': '#DOYLACARA. Con la trata no hay trato',
+            'duration': 50,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,

From 4647fd89103dc5ded16dc069828a2cf38216093c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:49:26 +0600
Subject: [PATCH 1328/2145] [telecinco] Fix typo

---
 youtube_dl/extractor/telecinco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index b0bf9ab8d..2c8e9b941 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class TelecincoIE(InfoExtractor):
-    IE_DESC = 'telecinco.es, cuatro.es and mediaset.es'
+    IE_DESC = 'telecinco.es, cuatro.com and mediaset.es'
     _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{

From 659ffe204c946b977266aa373110fe50a2763066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Sep 2015 22:58:49 +0600
Subject: [PATCH 1329/2145] [divxstage] Remove extractor

DivxStage is offline for quite some time already
---
 youtube_dl/extractor/__init__.py  |  1 -
 youtube_dl/extractor/divxstage.py | 27 ---------------------------
 2 files changed, 28 deletions(-)
 delete mode 100644 youtube_dl/extractor/divxstage.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 74b7df463..c16e9e3d2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -138,7 +138,6 @@ from .dump import DumpIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
-from .divxstage import DivxStageIE
 from .dropbox import DropboxIE
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
diff --git a/youtube_dl/extractor/divxstage.py b/youtube_dl/extractor/divxstage.py
deleted file mode 100644
index b88379e06..000000000
--- a/youtube_dl/extractor/divxstage.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class DivxStageIE(NovaMovIE):
-    IE_NAME = 'divxstage'
-    IE_DESC = 'DivxStage'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'divxstage\.(?:eu|net|ch|co|at|ag|to)'}
-
-    _HOST = 'www.divxstage.eu'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _TITLE_REGEX = r'<div class="video_det">\s*<strong>([^<]+)</strong>'
-    _DESCRIPTION_REGEX = r'<div class="video_det">\s*<strong>[^<]+</strong>\s*<p>([^<]+)</p>'
-
-    _TEST = {
-        'url': 'http://www.divxstage.eu/video/57f238e2e5e01',
-        'md5': '63969f6eb26533a1968c4d325be63e72',
-        'info_dict': {
-            'id': '57f238e2e5e01',
-            'ext': 'flv',
-            'title': 'youtubedl test video',
-            'description': 'This is a test video for youtubedl.',
-        }
-    }

From 1806a75415a3b62b07ab3fd282f915c6af14dfd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Sep 2015 22:59:15 +0600
Subject: [PATCH 1330/2145] [ok] Capture error message

---
 youtube_dl/extractor/odnoklassniki.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 66520c2c5..0cd96c10a 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
+    ExtractorError,
     unified_strdate,
     int_or_none,
     qualities,
@@ -72,6 +73,12 @@ class OdnoklassnikiIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://ok.ru/video/%s' % video_id, video_id)
 
+        error = self._search_regex(
+            r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
+            webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         player = self._parse_json(
             unescapeHTML(self._search_regex(
                 r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,

From 131d05033bfe3f5a8ef3241489a44dc5c1e03830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Sep 2015 22:59:32 +0600
Subject: [PATCH 1331/2145] [ok] Skip test

---
 youtube_dl/extractor/odnoklassniki.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 0cd96c10a..ccc88cfb1 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -29,6 +29,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
         },
+        'skip': 'Video has been blocked',
     }, {
         # metadataUrl
         'url': 'http://ok.ru/video/63567059965189-0',

From 8a7bbd160641dbbc13f68aaa26dfa49b71483a8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 17 Sep 2015 15:22:19 +0200
Subject: [PATCH 1332/2145] [postprocessor/ffmpeg] Always use the 'file:'
 protocol for filenames (fixes #6874)

If the filename contains ':' it is interpreted as a protocol.
It also handles filenames starting with '-'.
---
 youtube_dl/downloader/hls.py       |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b2436e732..7743e176a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -31,7 +31,7 @@ class HlsFD(FileDownloader):
         args = [
             encodeArgument(opt)
             for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
-        args.append(encodeFilename(tmpfilename, True))
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
 
         self._debug_cmd(args)
 
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1f723908b..4f320e124 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -135,7 +135,10 @@ class FFmpegPostProcessor(PostProcessor):
 
         files_cmd = []
         for path in input_paths:
-            files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
+            files_cmd.extend([
+                encodeArgument('-i'),
+                encodeFilename(self._ffmpeg_filename_argument(path), True)
+            ])
         cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
@@ -155,10 +158,10 @@ class FFmpegPostProcessor(PostProcessor):
         self.run_ffmpeg_multiple_files([path], out_path, opts)
 
     def _ffmpeg_filename_argument(self, fn):
-        # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
-        if fn.startswith('-'):
-            return './' + fn
-        return fn
+        # Always use 'file:' because the filename may contain ':' (ffmpeg
+        # interprets that as a protocol) or can start with '-' (-- is broken in
+        # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details)
+        return 'file:' + fn
 
 
 class FFmpegExtractAudioPP(FFmpegPostProcessor):

From 06368a232a9bc8b383db77b0087a2ab1af94a833 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Sep 2015 21:01:55 +0600
Subject: [PATCH 1333/2145] [playwire] Fix test

---
 youtube_dl/extractor/playwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
index bdc71017b..6d138ef25 100644
--- a/youtube_dl/extractor/playwire.py
+++ b/youtube_dl/extractor/playwire.py
@@ -19,7 +19,7 @@ class PlaywireIE(InfoExtractor):
             'id': '3353705',
             'ext': 'mp4',
             'title': 'S04_RM_UCL_Rus',
-            'thumbnail': 're:^http://.*\.png$',
+            'thumbnail': 're:^https?://.*\.png$',
             'duration': 145.94,
         },
     }, {

From d1c694ea4ad75eb14e02e27ba6db5be95a598f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Sep 2015 00:51:41 +0600
Subject: [PATCH 1334/2145] [bbc] Switch extraction to hq iptv hls (Closes
 #2732, closes #3023, closes #4828, closes #6803, closes #6855)

---
 youtube_dl/extractor/bbc.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index abc5a44a1..42526357a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -21,6 +21,7 @@ class BBCCoUkIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _MEDIASELECTOR_URLS = [
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
 
@@ -189,6 +190,12 @@ class BBCCoUkIE(InfoExtractor):
             # Skip DASH until supported
             elif transfer_format == 'dash':
                 pass
+            elif transfer_format == 'hls':
+                m3u8_formats = self._extract_m3u8_formats(
+                    href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=supplier, fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             # Direct link
             else:
                 formats.append({

From 1d67c9664099c261b7752cb56773e423d410fae6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Sep 2015 22:09:33 +0100
Subject: [PATCH 1335/2145] [shahid] fix api request url

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 6e9903d5e..3fd545373 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -81,7 +81,7 @@ class ShahidIE(InfoExtractor):
                 compat_urllib_parse.urlencode({
                     'apiKey': 'sh@hid0nlin3',
                     'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
-                }).encode('utf-8')),
+                })),
             video_id, 'Downloading video JSON')
 
         video = video[api_vars['playerType']]

From 5de5ab89b4136e7f2191f50d8d69587b90e0c3bb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Sep 2015 22:10:36 +0100
Subject: [PATCH 1336/2145] [shahid] change test ext

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 3fd545373..f76fb12c0 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -16,7 +16,7 @@ class ShahidIE(InfoExtractor):
         'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
         'info_dict': {
             'id': '90574',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
             'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
             'duration': 2972,

From 6c91a5a7f5408cf666f3ee40b53c0d9e42521b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:16:12 +0600
Subject: [PATCH 1337/2145] [extractor/generic] Fix following redirect in
 Refresh HTTP header on python 2

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec748ed9f..8881a8a23 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 
 import os
 import re
+import sys
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
@@ -230,6 +231,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': False,
             }
         },
+        {
+            # redirect in Refresh HTTP header
+            'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
+            'info_dict': {
+                'id': 'pO8h3EaFRdo',
+                'ext': 'mp4',
+                'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
+                'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
+                'upload_date': '20150917',
+                'uploader_id': 'brtvofficial',
+                'uploader': 'Boiler Room',
+            },
+            'params': {
+                'skip_download': False,
+            },
+        },
         {
             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
@@ -1808,6 +1825,9 @@ class GenericIE(InfoExtractor):
                 # Look also in Refresh HTTP header
                 refresh_header = head_response.headers.get('Refresh')
                 if refresh_header:
+                    # In python 2 response HTTP headers are bytestrings
+                    if sys.version_info < (3, 0) and isinstance(refresh_header, str):
+                        refresh_header = refresh_header.decode('iso-8859-1')
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))

From f817adc4689a2064fcab733d6aebf83fd0e2cff7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:31:23 +0600
Subject: [PATCH 1338/2145] [youtube:history] Disable extractor until #6893 is
 investigated

Wiped out part of my history as well
---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c16e9e3d2..b93156232 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -808,7 +808,7 @@ from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeFavouritesIE,
-    YoutubeHistoryIE,
+    #YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From 393ca8c94d1adb1490b23265370ce69043b92546 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:45:19 +0600
Subject: [PATCH 1339/2145] [arte:+7] Look for json vp url in iframe (Closes
 #6895)

---
 youtube_dl/extractor/arte.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 76de24477..9ecb6786c 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -4,6 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     find_xpath_attr,
     unified_strdate,
@@ -77,7 +81,13 @@ class ArteTVPlus7IE(InfoExtractor):
     def _extract_from_webpage(self, webpage, video_id, lang):
         json_url = self._html_search_regex(
             [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'],
-            webpage, 'json vp url')
+            webpage, 'json vp url', default=None)
+        if not json_url:
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
+                webpage, 'iframe url', group='iframe url')
+            json_url = compat_parse_qs(
+                compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)
 
     def _extract_from_json_url(self, json_url, video_id, lang):

From 5e39123b3b6fd740d133fd45824b49b4bc674fa7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:47:03 +0600
Subject: [PATCH 1340/2145] [arte:+7] Fix typo

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 9ecb6786c..2a00da3ee 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -85,7 +85,7 @@ class ArteTVPlus7IE(InfoExtractor):
         if not json_url:
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
-                webpage, 'iframe url', group='iframe url')
+                webpage, 'iframe url', group='url')
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From f005f96ea594a82969885ecd124b094c7e09949a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 20 Sep 2015 12:23:13 +0200
Subject: [PATCH 1341/2145] [youtube:history] Explain why it has disabled and
 skip test

---
 test/test_all_urls.py            | 2 +-
 youtube_dl/extractor/__init__.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a9db42b30..a929afd15 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -99,7 +99,7 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_keywords(self):
         self.assertMatch(':ytsubs', ['youtube:subscriptions'])
         self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
-        self.assertMatch(':ythistory', ['youtube:history'])
+        # self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
 
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b93156232..2529d8657 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -808,6 +808,8 @@ from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeFavouritesIE,
+    # disabled because it can wipe the watch history (see #6893)
+    # remember to uncumment test in test/test_all_urls when it's fixed
     #YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,

From 5a1a2e94548f25b5fd540090af3f32a1d875f6b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 21:08:29 +0600
Subject: [PATCH 1342/2145] [utils] Fix kwargs on old python 2 (Closes #6905)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 206dd56bc..1dc3153fd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -619,7 +619,7 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     # expected HTTP responses to meet HTTP/1.0 or later (see also
     # https://github.com/rg3/youtube-dl/issues/6727)
     if sys.version_info < (3, 0):
-        kwargs['strict'] = True
+        kwargs[b'strict'] = True
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')
     if source_address is not None:

From 5b4c54631a8888d75fa766c0bd6ec1822e6caec5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 23:12:40 +0600
Subject: [PATCH 1343/2145] [nfl] Add team domains (#6907)

---
 youtube_dl/extractor/nfl.py | 132 +++++++++++++++++++++++-------------
 1 file changed, 86 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index dc54634a5..fe143ef88 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -16,53 +16,93 @@ from ..utils import (
 
 class NFLIE(InfoExtractor):
     IE_NAME = 'nfl.com'
-    _VALID_URL = r'''(?x)https?://
-        (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
-        (?:.+?/)*
-        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
-    _TESTS = [
-        {
-            'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
-            'md5': '394ef771ddcd1354f665b471d78ec4c6',
-            'info_dict': {
-                'id': '0ap3000000398478',
-                'ext': 'mp4',
-                'title': 'Week 3: Redskins vs. Eagles highlights',
-                'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
-                'upload_date': '20140921',
-                'timestamp': 1411337580,
-                'thumbnail': 're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
-            'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
-            'info_dict': {
-                'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
-                'ext': 'mp4',
-                'title': 'LIVE: Post Game vs. Browns',
-                'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
-                'upload_date': '20131229',
-                'timestamp': 1388354455,
-                'thumbnail': 're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
-            'info_dict': {
-                'id': '0ap3000000467607',
-                'ext': 'mp4',
-                'title': 'Frustrations flare on the field',
-                'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
-                'timestamp': 1422850320,
-                'upload_date': '20150202',
-            },
-        },
-        {
-            'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
-            'only_matching': True,
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?P<host>
+                            (?:www\.)?
+                            (?:
+                                (?:
+                                    nfl|
+                                    buffalobills|
+                                    miamidolphins|
+                                    patriots|
+                                    newyorkjets|
+                                    baltimoreravens|
+                                    bengals|
+                                    clevelandbrowns|
+                                    steelers|
+                                    houstontexans|
+                                    colts|
+                                    jaguars|
+                                    titansonline|
+                                    denverbroncos|
+                                    kcchiefs|
+                                    raiders|
+                                    chargers|
+                                    dallascowboys|
+                                    giants|
+                                    philadelphiaeagles|
+                                    redskins|
+                                    chicagobears|
+                                    detroitlions|
+                                    packers|
+                                    vikings|
+                                    atlantafalcons|
+                                    panthers|
+                                    neworleanssaints|
+                                    buccaneers|
+                                    azcardinals|
+                                    stlouisrams|
+                                    49ers|
+                                    seahawks
+                                )\.com|
+                                .+?\.clubs\.nfl\.com
+                            )
+                        )/
+                        (?:.+?/)*
+                        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))
+                    '''
+    _TESTS = [{
+        'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
+        'md5': '394ef771ddcd1354f665b471d78ec4c6',
+        'info_dict': {
+            'id': '0ap3000000398478',
+            'ext': 'mp4',
+            'title': 'Week 3: Redskins vs. Eagles highlights',
+            'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
+            'upload_date': '20140921',
+            'timestamp': 1411337580,
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
-    ]
+    }, {
+        'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
+        'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
+        'info_dict': {
+            'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
+            'ext': 'mp4',
+            'title': 'LIVE: Post Game vs. Browns',
+            'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
+            'upload_date': '20131229',
+            'timestamp': 1388354455,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
+        'info_dict': {
+            'id': '0ap3000000467607',
+            'ext': 'mp4',
+            'title': 'Frustrations flare on the field',
+            'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
+            'timestamp': 1422850320,
+            'upload_date': '20150202',
+        },
+    }, {
+        'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.buffalobills.com/video/videos/Rex_Ryan_Show_World_Wide_Rex/b1dcfab2-3190-4bb1-bfc0-d6e603d6601a',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def prepend_host(host, url):

From 4423eba49b1ca5e9cfb78370ab0eee8f84fb72bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 23:45:01 +0600
Subject: [PATCH 1344/2145] [nfl] Add support for URLs without id (Closes
 #6907)

---
 youtube_dl/extractor/nfl.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index fe143ef88..55dc6107d 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -60,7 +60,7 @@ class NFLIE(InfoExtractor):
                             )
                         )/
                         (?:.+?/)*
-                        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))
+                        (?P<id>[^/#?&]+)
                     '''
     _TESTS = [{
         'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
@@ -96,6 +96,17 @@ class NFLIE(InfoExtractor):
             'timestamp': 1422850320,
             'upload_date': '20150202',
         },
+    }, {
+        'url': 'http://www.patriots.com/video/2015/09/18/10-days-gillette',
+        'md5': '4c319e2f625ffd0b481b4382c6fc124c',
+        'info_dict': {
+            'id': 'n-238346',
+            'ext': 'mp4',
+            'title': '10 Days at Gillette',
+            'description': 'md5:8cd9cd48fac16de596eadc0b24add951',
+            'timestamp': 1442618809,
+            'upload_date': '20150918',
+        },
     }, {
         'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
         'only_matching': True,
@@ -135,13 +146,14 @@ class NFLIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config_url = NFLIE.prepend_host(host, self._search_regex(
-            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL',
-            default='static/content/static/config/video/config.json'))
+            r'(?:(?:config|configURL)\s*:\s*|<nflcs:avplayer[^>]+data-config\s*=\s*)(["\'])(?P<config>.+?)\1',
+            webpage, 'config URL', default='static/content/static/config/video/config.json',
+            group='config'))
         # For articles, the id in the url is not the video id
         video_id = self._search_regex(
-            r'contentId\s*:\s*"([^"]+)"', webpage, 'video id', default=video_id)
-        config = self._download_json(config_url, video_id,
-                                     note='Downloading player config')
+            r'(?:<nflcs:avplayer[^>]+data-contentId\s*=\s*|contentId\s*:\s*)(["\'])(?P<id>.+?)\1',
+            webpage, 'video id', default=video_id, group='id')
+        config = self._download_json(config_url, video_id, 'Downloading player config')
         url_template = NFLIE.prepend_host(
             host, '{contentURLTemplate:}'.format(**config))
         video_data = self._download_json(

From 82c06a40acb41df77ee55acf8979eb5f0cfba4c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 23:54:05 +0600
Subject: [PATCH 1345/2145] Fix typo

---
 youtube_dl/extractor/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2529d8657..35d7f0dd2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -809,8 +809,8 @@ from .youtube import (
     YoutubeChannelIE,
     YoutubeFavouritesIE,
     # disabled because it can wipe the watch history (see #6893)
-    # remember to uncumment test in test/test_all_urls when it's fixed
-    #YoutubeHistoryIE,
+    # remember to uncomment test in test/test_all_urls when it's fixed
+    YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From 9f5e8d16b3c7a1a2731cbab9661a920fbe04e09f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Sep 2015 01:53:28 +0600
Subject: [PATCH 1346/2145] [youtube:history] Disable exractor

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 35d7f0dd2..030c00790 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -810,7 +810,7 @@ from .youtube import (
     YoutubeFavouritesIE,
     # disabled because it can wipe the watch history (see #6893)
     # remember to uncomment test in test/test_all_urls when it's fixed
-    YoutubeHistoryIE,
+    #YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From c6aa838b51484ad3d036db21ba4e9e834720a693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Sep 2015 21:28:02 +0600
Subject: [PATCH 1347/2145] [youtube:history] Enable exractor

---
 test/test_all_urls.py            | 2 +-
 youtube_dl/extractor/__init__.py | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a929afd15..a9db42b30 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -99,7 +99,7 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_keywords(self):
         self.assertMatch(':ytsubs', ['youtube:subscriptions'])
         self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
-        # self.assertMatch(':ythistory', ['youtube:history'])
+        self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
 
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 030c00790..c16e9e3d2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -808,9 +808,7 @@ from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeFavouritesIE,
-    # disabled because it can wipe the watch history (see #6893)
-    # remember to uncomment test in test/test_all_urls when it's fixed
-    #YoutubeHistoryIE,
+    YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From 92085e7099d3607ff512f241454d6f6f4535b05a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Sep 2015 22:26:23 +0100
Subject: [PATCH 1348/2145] [viewster] accept https links and fix api_token
 extraction and extract mp4 video link(fixes #6787)

---
 youtube_dl/extractor/viewster.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index cda02ba24..c68a38167 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class ViewsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
+    _VALID_URL = r'https?://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
     _TESTS = [{
         # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
@@ -74,8 +74,8 @@ class ViewsterIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # Get 'api_token' cookie
-        self._request_webpage(HEADRequest(url), video_id)
-        cookies = self._get_cookies(url)
+        self._request_webpage(HEADRequest('http://www.viewster.com/'), video_id)
+        cookies = self._get_cookies('http://www.viewster.com/')
         self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 
         info = self._download_json(
@@ -98,7 +98,7 @@ class ViewsterIE(InfoExtractor):
             return self.playlist_result(entries, video_id, title, description)
 
         formats = []
-        for media_type in ('application/f4m+xml', 'application/x-mpegURL'):
+        for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
             media = self._download_json(
                 'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
                 % (entry_id, compat_urllib_parse.quote(media_type)),
@@ -122,6 +122,8 @@ class ViewsterIE(InfoExtractor):
             else:
                 formats.append({
                     'url': video_url,
+                    'height': int_or_none(media.get('Height')),
+                    'width': int_or_none(media.get('Width')),
                 })
         self._sort_formats(formats)
 

From 8e97596b7b62fb35ccfeb37c0ce5cb008b217d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:47:56 +0600
Subject: [PATCH 1349/2145] [viewster] Extract height from bitrate and prefer
 mp4 videos

---
 youtube_dl/extractor/viewster.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index c68a38167..c6aa73c0e 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -120,11 +120,18 @@ class ViewsterIE(InfoExtractor):
                     fatal=False  # m3u8 sometimes fail
                 ))
             else:
-                formats.append({
+                format_id = media.get('Bitrate')
+                f = {
                     'url': video_url,
+                    'format_id': 'mp4-%s' % format_id,
                     'height': int_or_none(media.get('Height')),
                     'width': int_or_none(media.get('Width')),
-                })
+                    'preference': 1,
+                }
+                if format_id and not f['height']:
+                    f['height'] = int_or_none(self._search_regex(
+                        r'^(\d+)[pP]$', format_id, 'height', default=None))
+                formats.append(f)
         self._sort_formats(formats)
 
         synopsis = info.get('Synopsis', {})

From cb4e421901ce5fe26b457c33c141267c5c18a237 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:49:29 +0600
Subject: [PATCH 1350/2145] [voewster] Update tests

---
 youtube_dl/extractor/viewster.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index c6aa73c0e..8defc1800 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -20,10 +20,10 @@ class ViewsterIE(InfoExtractor):
     _TESTS = [{
         # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
-        'md5': '14d3cfffe66d57b41ae2d9c873416f01',
+        'md5': 'e642d1b27fcf3a4ffa79f194f5adde36',
         'info_dict': {
             'id': '1140-11855-000',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'The listening Project',
             'description': 'md5:bac720244afd1a8ea279864e67baa071',
             'timestamp': 1214870400,
@@ -33,10 +33,10 @@ class ViewsterIE(InfoExtractor):
     }, {
         # series episode, Type=Episode
         'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
-        'md5': 'd5434c80fcfdb61651cc2199a88d6ba3',
+        'md5': '9243079a8531809efe1b089db102c069',
         'info_dict': {
             'id': '1284-19427-001',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'The World and a Wall',
             'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
             'timestamp': 1428192000,

From c430802e32932868e24a6e43d0845df963320829 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:50:20 +0600
Subject: [PATCH 1351/2145] [extractor/common] Add raise_geo_restricted

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d694e818e..1e7db8a9b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -516,6 +516,12 @@ class InfoExtractor(object):
             '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
             expected=True)
 
+    @staticmethod
+    def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'):
+        raise ExtractorError(
+            '%s. You might want to use --proxy to workaround.' % msg,
+            expected=True)
+
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):

From cccedc1aa43f94ce4f8fa6f807a5301250c2213c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:52:41 +0600
Subject: [PATCH 1352/2145] [voewster] Detect series geo restriction

---
 youtube_dl/extractor/viewster.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 8defc1800..f8f4143c6 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
 )
 from ..utils import (
     determine_ext,
+    ExtractorError,
     int_or_none,
     parse_iso8601,
     HEADRequest,
@@ -86,9 +88,15 @@ class ViewsterIE(InfoExtractor):
 
         # unfinished serie has no Type
         if info.get('Type') in ['Serie', None]:
-            episodes = self._download_json(
-                'https://public-api.viewster.com/series/%s/episodes' % entry_id,
-                video_id, 'Downloading series JSON')
+            try:
+                episodes = self._download_json(
+                    'https://public-api.viewster.com/series/%s/episodes' % entry_id,
+                    video_id, 'Downloading series JSON')
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                    self.raise_geo_restricted()
+                else:
+                    raise
             entries = [
                 self.url_result(
                     'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')

From 9612f2339909f958f0c7ee7ccc370151ec28358d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:54:32 +0600
Subject: [PATCH 1353/2145] [viewster] Detect video geo restriction

---
 youtube_dl/extractor/viewster.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index f8f4143c6..7c4328efb 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -140,6 +140,10 @@ class ViewsterIE(InfoExtractor):
                     f['height'] = int_or_none(self._search_regex(
                         r'^(\d+)[pP]$', format_id, 'height', default=None))
                 formats.append(f)
+
+        if not formats and not info.get('LanguageSets') and not info.get('VODSettings'):
+            self.raise_geo_restricted()
+
         self._sort_formats(formats)
 
         synopsis = info.get('Synopsis', {})

From 7ce50a355c3a4cbd61f18fb16d8175a97f14d510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:55:04 +0600
Subject: [PATCH 1354/2145] [viewster] Add geo restricted tests

---
 youtube_dl/extractor/viewster.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 7c4328efb..487f2d744 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -63,6 +63,14 @@ class ViewsterIE(InfoExtractor):
             'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
         },
         'playlist_mincount': 16,
+    }, {
+        # geo restricted series
+        'url': 'https://www.viewster.com/serie/1280-18794-002/',
+        'only_matching': True,
+    }, {
+        # geo restricted video
+        'url': 'https://www.viewster.com/serie/1280-18794-002/what-is-extraterritoriality-lawo/',
+        'only_matching': True,
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'

From d0fed4ac028baa40b18082fd813ed93cd658ebe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 22:00:50 +0600
Subject: [PATCH 1355/2145] [viewster] Use tuple

---
 youtube_dl/extractor/viewster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 487f2d744..632e57fb4 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -95,7 +95,7 @@ class ViewsterIE(InfoExtractor):
         entry_id = info.get('Id') or info['id']
 
         # unfinished serie has no Type
-        if info.get('Type') in ['Serie', None]:
+        if info.get('Type') in ('Serie', None):
             try:
                 episodes = self._download_json(
                     'https://public-api.viewster.com/series/%s/episodes' % entry_id,

From de3fc356e1597b31c8f0a55a7bf7e201ae436c66 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Sep 2015 14:01:12 +0100
Subject: [PATCH 1356/2145] [ninegag] fix _VALID_URL regex and handle the use
 of other external providers

---
 youtube_dl/extractor/ninegag.py | 43 ++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 7f842b5c2..eee65873a 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -9,17 +9,12 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'''(?x)^https?://(?:www\.)?9gag\.tv/
-        (?:
-            v/(?P<numid>[0-9]+)|
-            p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)
-        )
-    '''
+    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)'
 
     _TESTS = [{
-        "url": "http://9gag.tv/v/1912",
+        "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
         "info_dict": {
-            "id": "1912",
+            "id": "Kk2X5",
             "ext": "mp4",
             "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
             "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
@@ -31,7 +26,7 @@ class NineGagIE(InfoExtractor):
         },
         'add_ie': ['Youtube']
     }, {
-        'url': 'http://9gag.tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
+        'url': 'http://9gag.com/tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
         'info_dict': {
             'id': 'KklwM',
             'ext': 'mp4',
@@ -42,19 +37,39 @@ class NineGagIE(InfoExtractor):
             'upload_date': '20140401',
             'uploader_id': 'krishnashenoi93',
         },
+        'add_ie': ['Youtube']
     }]
+    _EXTERNAL_VIDEO_PROVIDER = {
+        '1': {
+            'url': '%s',
+            'ie_key': 'Youtube',
+        },
+        '2': {
+            'url': 'http://player.vimeo.com/video/%s',
+            'ie_key': 'Vimeo',
+        },
+        '3': {
+            'url': 'http://instagram.com/p/%s',
+            'ie_key': 'Instagram',
+        },
+        '4': {
+            'url': 'http://vine.co/v/%s',
+            'ie_key': 'Vine',
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('numid') or mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
         webpage = self._download_webpage(url, display_id)
 
         post_view = json.loads(self._html_search_regex(
             r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
 
-        youtube_id = post_view['videoExternalId']
+        external_video_id = post_view['videoExternalId']
+        external_video_provider = post_view['videoExternalProvider']
         title = post_view['title']
         description = post_view['description']
         view_count = str_to_int(post_view['externalView'])
@@ -62,8 +77,8 @@ class NineGagIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'url': youtube_id,
-            'ie_key': 'Youtube',
+            'url': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id,
+            'ie_key': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'],
             'id': video_id,
             'display_id': display_id,
             'title': title,

From 6b8ce312e3cb7d81e949cd5c64ad2a824d27830b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Sep 2015 19:20:18 +0100
Subject: [PATCH 1357/2145] [ninegag] extract source url

---
 youtube_dl/extractor/ninegag.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index eee65873a..00bf98ef5 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -68,8 +68,13 @@ class NineGagIE(InfoExtractor):
         post_view = json.loads(self._html_search_regex(
             r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
 
-        external_video_id = post_view['videoExternalId']
-        external_video_provider = post_view['videoExternalProvider']
+        ie_key = None
+        source_url = post_view.get('sourceUrl')
+        if not source_url or source_url == '':
+            external_video_id = post_view['videoExternalId']
+            external_video_provider = post_view['videoExternalProvider']
+            source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
+            ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
         title = post_view['title']
         description = post_view['description']
         view_count = str_to_int(post_view['externalView'])
@@ -77,8 +82,8 @@ class NineGagIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'url': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id,
-            'ie_key': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'],
+            'url': source_url,
+            'ie_key': ie_key,
             'id': video_id,
             'display_id': display_id,
             'title': title,

From da9f18083596d0132d12652acd0bd8983c70c058 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Sep 2015 20:28:00 +0100
Subject: [PATCH 1358/2145] [ninegag] remove unnecessary condition

---
 youtube_dl/extractor/ninegag.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 00bf98ef5..0a2725c65 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -70,7 +70,7 @@ class NineGagIE(InfoExtractor):
 
         ie_key = None
         source_url = post_view.get('sourceUrl')
-        if not source_url or source_url == '':
+        if not source_url:
             external_video_id = post_view['videoExternalId']
             external_video_provider = post_view['videoExternalProvider']
             source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id

From e28c794699596912092635014b041d0af888fd08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:40:06 +0600
Subject: [PATCH 1359/2145] [9gag] Make display_id optional

---
 youtube_dl/extractor/ninegag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 0a2725c65..6103c7517 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -9,7 +9,7 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)'
+    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
         "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
@@ -61,7 +61,7 @@ class NineGagIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or video_id
 
         webpage = self._download_webpage(url, display_id)
 

From c3a4e2ec40f9a90d4ae8991e6764641c744ec8fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:41:44 +0600
Subject: [PATCH 1360/2145] [9gag] Remove redundant test

---
 youtube_dl/extractor/ninegag.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 6103c7517..34879d1c8 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -26,18 +26,8 @@ class NineGagIE(InfoExtractor):
         },
         'add_ie': ['Youtube']
     }, {
-        'url': 'http://9gag.com/tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
-        'info_dict': {
-            'id': 'KklwM',
-            'ext': 'mp4',
-            'display_id': 'alternate-banned-opening-scene-of-gravity',
-            "description": "While Gravity was a pretty awesome movie already, YouTuber Krishna Shenoi came up with a way to improve upon it, introducing a much better solution to Sandra Bullock's seemingly endless tumble in space. The ending is priceless.",
-            'title': "Banned Opening Scene Of \"Gravity\" That Changes The Whole Movie",
-            'uploader': 'Krishna Shenoi',
-            'upload_date': '20140401',
-            'uploader_id': 'krishnashenoi93',
-        },
-        'add_ie': ['Youtube']
+        'url': 'http://9gag.com/tv/p/KklwM',
+        'only_matching': True,
     }]
     _EXTERNAL_VIDEO_PROVIDER = {
         '1': {

From 6400f8ec0f20011a7e39da62e0a3e55e0fd2759a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:43:26 +0600
Subject: [PATCH 1361/2145] [9gag] Allow old .tv domain

There are still references to it in webpage's source
---
 youtube_dl/extractor/ninegag.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 34879d1c8..d157b0d10 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -9,7 +9,7 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
+    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
         "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
@@ -28,6 +28,9 @@ class NineGagIE(InfoExtractor):
     }, {
         'url': 'http://9gag.com/tv/p/KklwM',
         'only_matching': True,
+    }, {
+        'url': 'http://9gag.tv/p/Kk2X5',
+        'only_matching': True,
     }]
     _EXTERNAL_VIDEO_PROVIDER = {
         '1': {

From 5600e214c3b63e3e2a0862bea230026c02073d7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:44:38 +0600
Subject: [PATCH 1362/2145] [9gag] Make post view regex more robust

---
 youtube_dl/extractor/ninegag.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index d157b0d10..692cc3368 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -59,7 +59,8 @@ class NineGagIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         post_view = json.loads(self._html_search_regex(
-            r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
+            r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
+            webpage, 'post view'))
 
         ie_key = None
         source_url = post_view.get('sourceUrl')

From 8ca2e93e1ab3485946df19d56a87e56699b2a712 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:46:40 +0600
Subject: [PATCH 1363/2145] [9gag] Relax optional fields

---
 youtube_dl/extractor/ninegag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 692cc3368..6daae7318 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -70,8 +70,8 @@ class NineGagIE(InfoExtractor):
             source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
             ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
         title = post_view['title']
-        description = post_view['description']
-        view_count = str_to_int(post_view['externalView'])
+        description = post_view.get('description')
+        view_count = str_to_int(post_view.get('externalView'))
         thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')
 
         return {

From c659022b5cabebaba3275df2d0f4ae97b468eac8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:48:13 +0600
Subject: [PATCH 1364/2145] [9gag] Modernize

---
 youtube_dl/extractor/ninegag.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 6daae7318..d8e103189 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -58,9 +58,11 @@ class NineGagIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        post_view = json.loads(self._html_search_regex(
-            r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
-            webpage, 'post view'))
+        post_view = self._parse_json(
+            self._search_regex(
+                r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
+                webpage, 'post view'),
+            display_id)
 
         ie_key = None
         source_url = post_view.get('sourceUrl')

From 8ea6bd2802a0dbb860187bb5aaf687585ff7c1c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:55:16 +0600
Subject: [PATCH 1365/2145] [9gag] Add vimeo test

---
 youtube_dl/extractor/ninegag.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index d8e103189..0ef6a6c65 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import str_to_int
@@ -22,9 +21,21 @@ class NineGagIE(InfoExtractor):
             'uploader': 'CompilationChannel',
             'upload_date': '20131110',
             "view_count": int,
-            "thumbnail": "re:^https?://",
         },
-        'add_ie': ['Youtube']
+        'add_ie': ['Youtube'],
+    }, {
+        'url': 'http://9gag.com/tv/p/aKolP3',
+        'info_dict': {
+            'id': 'aKolP3',
+            'ext': 'mp4',
+            'title': 'This Guy Travelled 11 countries In 44 days Just To Make This Amazing Video',
+            'description': "I just saw more in 1 minute than I've seen in 1 year. This guy's video is epic!!",
+            'uploader_id': 'rickmereki',
+            'uploader': 'Rick Mereki',
+            'upload_date': '20110803',
+            'view_count': int,
+        },
+        'add_ie': ['Vimeo'],
     }, {
         'url': 'http://9gag.com/tv/p/KklwM',
         'only_matching': True,

From d8fef8faacad4f3b9d13e20df4ee9344ada3c68d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:56:26 +0600
Subject: [PATCH 1366/2145] [9gag] Quotes consistency

---
 youtube_dl/extractor/ninegag.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 0ef6a6c65..4adff197e 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -11,16 +11,16 @@ class NineGagIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
-        "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
-        "info_dict": {
-            "id": "Kk2X5",
-            "ext": "mp4",
-            "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
-            "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
+        'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
+        'info_dict': {
+            'id': 'Kk2X5',
+            'ext': 'mp4',
+            'description': 'This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)',
+            'title': '\"People Are Awesome 2013\" Is Absolutely Awesome',
             'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
             'uploader': 'CompilationChannel',
             'upload_date': '20131110',
-            "view_count": int,
+            'view_count': int,
         },
         'add_ie': ['Youtube'],
     }, {

From 78f9fb902b36c2a12dfe50d9724ac470b03c87d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:58:47 +0600
Subject: [PATCH 1367/2145] [9gag] Support embed URLs

---
 youtube_dl/extractor/ninegag.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 4adff197e..a06d38afd 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -8,7 +8,7 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
+    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/(?:p|embed)/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
         'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
@@ -42,7 +42,11 @@ class NineGagIE(InfoExtractor):
     }, {
         'url': 'http://9gag.tv/p/Kk2X5',
         'only_matching': True,
+    }, {
+        'url': 'http://9gag.com/tv/embed/a5Dmvl',
+        'only_matching': True,
     }]
+
     _EXTERNAL_VIDEO_PROVIDER = {
         '1': {
             'url': '%s',

From b942db3dc3d51db0f24ac98ada861d8e2b3451db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 22 Sep 2015 22:41:53 +0200
Subject: [PATCH 1368/2145] release 2015.09.22

---
 docs/supportedsites.md | 17 ++++++++++-------
 youtube_dl/version.py  |  2 +-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 66091e6be..ab153af6b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -122,7 +122,6 @@
  - **defense.gouv.fr**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Discovery**
- - **divxstage**: DivxStage
  - **Dotsub**
  - **DouyuTV**: 斗鱼
  - **dramafever**
@@ -286,7 +285,7 @@
  - **Minhateca**
  - **MinistryGrid**
  - **miomio.tv**
- - **mitele.es**
+ - **MiTele**: mitele.es
  - **mixcloud**
  - **MLB**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
@@ -317,7 +316,6 @@
  - **Myvi**
  - **myvideo**
  - **MyVidster**
- - **N-JOY**
  - **n-tv.de**
  - **NationalGeographic**
  - **Naver**
@@ -326,7 +324,9 @@
  - **NBCNews**
  - **NBCSports**
  - **NBCSportsVPlayer**
- - **ndr**: NDR.de - Mediathek
+ - **ndr**: NDR.de - Norddeutscher Rundfunk
+ - **ndr:embed**
+ - **ndr:embed:base**
  - **NDTV**
  - **NerdCubedFeed**
  - **Nerdist**
@@ -349,12 +349,16 @@
  - **nhl.com:videocenter**: NHL videocenter category
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
+ - **njoy**: N-JOY
+ - **njoy:embed**
  - **Noco**
  - **Normalboots**
  - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **novamov**: NovaMov
- - **Nowness**
+ - **nowness**
+ - **nowness:playlist**
+ - **nowness:series**
  - **NowTV**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
@@ -375,7 +379,6 @@
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
- - **OpenFilm**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
@@ -530,7 +533,7 @@
  - **techtv.mit.edu**
  - **ted**
  - **TeleBruxelles**
- - **telecinco.es**
+ - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0cc7411f2..7ef4f2755 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.09'
+__version__ = '2015.09.22'

From f1028194636e7acafb29fb38244cc7b1347d9313 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 02:46:24 +0600
Subject: [PATCH 1369/2145] [downloader/hls] Pass http headers to downloader

---
 youtube_dl/downloader/hls.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 7743e176a..a62d2047b 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -28,9 +28,18 @@ class HlsFD(FileDownloader):
             return False
         ffpp.check_version()
 
-        args = [
-            encodeArgument(opt)
-            for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args = [ffpp.executable, '-y']
+
+        if info_dict['http_headers']:
+            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
+            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            args += [
+                '-headers',
+                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items())]
+
+        args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+
+        args = [encodeArgument(opt) for opt in args]
         args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
 
         self._debug_cmd(args)

From 53407e3f383ed80c67db9e06b8c3480257aa3184 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 14:02:13 +0100
Subject: [PATCH 1370/2145] [brightcove] fix streaming_src extraction

---
 youtube_dl/extractor/brightcove.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a07c0888f..e4a7befee 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -413,7 +413,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
             if source_type == 'application/x-mpegURL':
                 formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
             else:
-                src = source.get('src')
+                src = source.get('src') or source.get('streaming_src')
                 if src:
                     formats.append({
                         'url': src,
@@ -424,8 +424,6 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
                         'container': source.get('container'),
                         'vcodec': source.get('container'),
                     })
-                else:
-                    formats.extend(self._extract_f4m_formats(source.get('streaming_src'), video_id))
 
         self._sort_formats(formats)
 

From eb11cbe8674705647d6bd2947a44e08543663633 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 19:54:40 +0600
Subject: [PATCH 1371/2145] [soundcloud] Update client id (Closes #6930)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index ed5dcc0d3..2b60d354a 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -113,7 +113,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
+    _CLIENT_ID = '02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     def report_resolve(self, video_id):

From 57565375c85ff6adb11beb961fd61d6bdd023ec1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Sep 2015 22:22:04 +0800
Subject: [PATCH 1372/2145] [iqiyi] Fix extraction (fixes #6878)

---
 youtube_dl/extractor/iqiyi.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 393e67e35..2ce6627a4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -95,6 +95,10 @@ class IqiyiIE(InfoExtractor):
         ('10', 'h1'),
     ]
 
+    @staticmethod
+    def md5_text(text):
+        return hashlib.md5(text.encode('utf-8')).hexdigest()
+
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
@@ -179,6 +183,7 @@ class IqiyiIE(InfoExtractor):
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))
+        tail = tm + tvid
         param = {
             'key': 'fvip',
             'src': hashlib.md5(b'youtube-dl').hexdigest(),
@@ -186,13 +191,11 @@ class IqiyiIE(InfoExtractor):
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': hashlib.md5(
-                (enc_key + tm + tvid).encode('utf8')).hexdigest(),
+            'enc': self.md5_text((enc_key + tail)[1:64:2] + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
-            'authkey': hashlib.md5(
-                (tm + tvid).encode('utf8')).hexdigest()
+            'authkey': self.md5_text(self.md5_text('') + tail),
         }
 
         api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
@@ -201,7 +204,8 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        enc_key = '3601ba290e4f4662848c710e2122007e'  # last update at 2015-08-10 for Zombie
+        # TODO: automatic key extraction
+        enc_key = 'eac64f22daf001da6ba9aa8da4d501508bbe90a4d4091fea3b0582a85b38c2cc'  # last update at 2015-09-23-23 for Zombie::bite
         return enc_key
 
     def _real_extract(self, url):

From 19f93d906e29e9a505d4bf5d286d75224c342c37 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Sep 2015 22:25:16 +0800
Subject: [PATCH 1373/2145] [iqiyi] Use md5_text for all MD5 calls

---
 youtube_dl/extractor/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2ce6627a4..ce1ab3820 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -125,7 +125,7 @@ class IqiyiIE(InfoExtractor):
                 note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
             )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
-            return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
+            return self.md5_text(t + mg + x)
 
         video_urls_dict = {}
         for format_item in data['vp']['tkl'][0]['vs']:
@@ -186,7 +186,7 @@ class IqiyiIE(InfoExtractor):
         tail = tm + tvid
         param = {
             'key': 'fvip',
-            'src': hashlib.md5(b'youtube-dl').hexdigest(),
+            'src': self.md5_text('youtube-dl'),
             'tvId': tvid,
             'vid': video_id,
             'vinfo': 1,

From 4395ca2e04f38c110259270fa4dfc4d9814aa926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 19:56:54 +0600
Subject: [PATCH 1374/2145] [xhamster] Fix title extraction (Closes #6944)

---
 youtube_dl/extractor/xhamster.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 97315750f..f12fe13b1 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -63,7 +63,9 @@ class XHamsterIE(InfoExtractor):
         mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
         webpage = self._download_webpage(mrss_url, video_id)
 
-        title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>', webpage, 'title')
+        title = self._html_search_regex(
+            [r'<title>(?P<title>.+?)(?:, Free Porn: xHamster| - xHamster\.com)</title>',
+             r'<h1>([^<]+)</h1>'], webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From 05b476a27087a45b8d418607123d8db62bf1770f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 23:38:53 +0600
Subject: [PATCH 1375/2145] [vidme] Prefer non clip (Closes #6924)

---
 youtube_dl/extractor/vidme.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 9a794e609..078d283b2 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -119,6 +119,7 @@ class VidmeIE(InfoExtractor):
             'url': f['uri'],
             'width': int_or_none(f.get('width')),
             'height': int_or_none(f.get('height')),
+            'preference': 0 if f.get('type', '').endswith('clip') else 1,
         } for f in video.get('formats', []) if f.get('uri')]
         self._sort_formats(formats)
 

From 9fbd4b35a27a83055c3e170ab32f2b3e56f9616e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 23:48:23 +0600
Subject: [PATCH 1376/2145] [nhl] Add support for embedded URLs

---
 youtube_dl/extractor/nhl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 279b18386..970a11f7c 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -72,7 +72,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P<id>[-0-9a-zA-Z,]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console|embed)?(?:\?(?:.*?[?&])?)(?:id|hlg|playlist)=(?P<id>[-0-9a-zA-Z,]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -136,6 +136,9 @@ class NHLIE(NHLBaseInfoExtractor):
         'params': {
             'skip_download': True,  # Requires rtmpdump
         }
+    }, {
+        'url': 'http://video.nhl.com/videocenter/embed?playlist=836127',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9c58885c70af75655288933220a99b4c4215ab4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 23:54:16 +0600
Subject: [PATCH 1377/2145] [nhl:news] Add support for iframe embeds (Closes
 #6941)

---
 youtube_dl/extractor/nhl.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 970a11f7c..e98a5ef89 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -149,9 +149,9 @@ class NHLIE(NHLBaseInfoExtractor):
 class NHLNewsIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com:news'
     IE_DESC = 'NHL news'
-    _VALID_URL = r'https?://(?:www\.)?nhl\.com/ice/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:.+?\.)?nhl\.com/(?:ice|club)/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nhl.com/ice/news.htm?id=750727',
         'md5': '4b3d1262e177687a3009937bd9ec0be8',
         'info_dict': {
@@ -162,13 +162,26 @@ class NHLNewsIE(NHLBaseInfoExtractor):
             'duration': 37,
             'upload_date': '20150128',
         },
-    }
+    }, {
+        # iframe embed
+        'url': 'http://sabres.nhl.com/club/news.htm?id=780189',
+        'md5': '9f663d1c006c90ac9fb82777d4294e12',
+        'info_dict': {
+            'id': '836127',
+            'ext': 'mp4',
+            'title': 'Morning Skate: OTT vs. BUF (9/23/15)',
+            'description': "Brian Duff chats with Tyler Ennis prior to Buffalo's first preseason home game.",
+            'duration': 93,
+            'upload_date': '20150923',
+        },
+    }]
 
     def _real_extract(self, url):
         news_id = self._match_id(url)
         webpage = self._download_webpage(url, news_id)
         video_id = self._search_regex(
-            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'"],
+            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'",
+             r'<iframe[^>]+src=["\']https?://video.*?\.nhl\.com/videocenter/embed\?.*\bplaylist=(\d+)'],
             webpage, 'video id')
         return self._real_extract_video(video_id)
 

From 47024eb564fd4047e362680f0a68304d1df79495 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 19:49:10 +0100
Subject: [PATCH 1378/2145] [hostingbulk] remove extractor

---
 youtube_dl/extractor/__init__.py    |  1 -
 youtube_dl/extractor/hostingbulk.py | 80 -----------------------------
 2 files changed, 81 deletions(-)
 delete mode 100644 youtube_dl/extractor/hostingbulk.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c16e9e3d2..7272859db 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -228,7 +228,6 @@ from .historicfilms import HistoricFilmsIE
 from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
-from .hostingbulk import HostingBulkIE
 from .hotnewhiphop import HotNewHipHopIE
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
deleted file mode 100644
index a3154cfde..000000000
--- a/youtube_dl/extractor/hostingbulk.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    urlencode_postdata,
-)
-
-
-class HostingBulkIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?hostingbulk\.com/
-        (?:embed-)?(?P<id>[A-Za-z0-9]{12})(?:-\d+x\d+)?\.html'''
-    _FILE_DELETED_REGEX = r'<b>File Not Found</b>'
-    _TEST = {
-        'url': 'http://hostingbulk.com/n0ulw1hv20fm.html',
-        'md5': '6c8653c8ecf7ebfa83b76e24b7b2fe3f',
-        'info_dict': {
-            'id': 'n0ulw1hv20fm',
-            'ext': 'mp4',
-            'title': 'md5:5afeba33f48ec87219c269e054afd622',
-            'filesize': 6816081,
-            'thumbnail': 're:^http://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://hostingbulk.com/{0:}.html'.format(video_id)
-
-        # Custom request with cookie to set language to English, so our file
-        # deleted regex would work.
-        request = compat_urllib_request.Request(
-            url, headers={'Cookie': 'lang=english'})
-        webpage = self._download_webpage(request, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        title = self._html_search_regex(r'<h3>(.*?)</h3>', webpage, 'title')
-        filesize = int_or_none(
-            self._search_regex(
-                r'<small>\((\d+)\sbytes?\)</small>',
-                webpage,
-                'filesize',
-                fatal=False
-            )
-        )
-        thumbnail = self._search_regex(
-            r'<img src="([^"]+)".+?class="pic"',
-            webpage, 'thumbnail', fatal=False)
-
-        fields = self._hidden_inputs(webpage)
-
-        request = compat_urllib_request.Request(url, urlencode_postdata(fields))
-        request.add_header('Content-type', 'application/x-www-form-urlencoded')
-        response = self._request_webpage(request, video_id,
-                                         'Submiting download request')
-        video_url = response.geturl()
-
-        formats = [{
-            'format_id': 'sd',
-            'filesize': filesize,
-            'url': video_url,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 545a23f11bbdf27d6d204225c578512af7184e0b Mon Sep 17 00:00:00 2001
From: atomic83 <atomic83@riseup.net>
Date: Thu, 24 Sep 2015 23:05:32 +0200
Subject: [PATCH 1379/2145] More title extraction fixing.

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f12fe13b1..8938c0e45 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -64,7 +64,7 @@ class XHamsterIE(InfoExtractor):
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(
-            [r'<title>(?P<title>.+?)(?:, Free Porn: xHamster| - xHamster\.com)</title>',
+            [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)</title>',
              r'<h1>([^<]+)</h1>'], webpage, 'title')
 
         # Only a few videos have an description

From 857421024daf810e92036149cc02bcf1c337da5c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 21:55:44 +0100
Subject: [PATCH 1380/2145] [iconosquare] fix info extraction

---
 youtube_dl/extractor/iconosquare.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 70e4c0d41..4fff8c0b3 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    get_element_by_id,
+)
 
 
 class IconosquareIE(InfoExtractor):
@@ -12,7 +15,7 @@ class IconosquareIE(InfoExtractor):
         'info_dict': {
             'id': '522207370455279102_24101272',
             'ext': 'mp4',
-            'title': 'Instagram media by @aguynamedpatrick (Patrick Janelle)',
+            'title': 'A little over a year ago, I posted my first #dailycortado, a drink introduced to...',
             'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
             'timestamp': 1376471991,
             'upload_date': '20130814',
@@ -29,8 +32,7 @@ class IconosquareIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         media = self._parse_json(
-            self._search_regex(
-                r'window\.media\s*=\s*({.+?});\n', webpage, 'media'),
+            get_element_by_id('mediaJson', webpage),
             video_id)
 
         formats = [{
@@ -42,7 +44,7 @@ class IconosquareIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._html_search_regex(
-            r'<title>(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)</title>',
+            r'<title>(.+?)</title>',
             webpage, 'title')
 
         timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))

From d4364f30bd39528b4da487799380737c330e88c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 04:44:52 +0600
Subject: [PATCH 1381/2145] [iconosquare] Revert title (Closes #6954)

---
 youtube_dl/extractor/iconosquare.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 4fff8c0b3..bb69c7a0f 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     get_element_by_id,
+    remove_end,
 )
 
 
@@ -15,7 +16,7 @@ class IconosquareIE(InfoExtractor):
         'info_dict': {
             'id': '522207370455279102_24101272',
             'ext': 'mp4',
-            'title': 'A little over a year ago, I posted my first #dailycortado, a drink introduced to...',
+            'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)',
             'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
             'timestamp': 1376471991,
             'upload_date': '20130814',
@@ -43,9 +44,7 @@ class IconosquareIE(InfoExtractor):
         } for format_id, f in media['videos'].items()]
         self._sort_formats(formats)
 
-        title = self._html_search_regex(
-            r'<title>(.+?)</title>',
-            webpage, 'title')
+        title = remove_end(self._og_search_title(webpage), ' - via Iconosquare')
 
         timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))
         description = media.get('caption', {}).get('text')

From 9b166fc1f8039fe3f3632c40848ce590ade9f3fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 04:45:31 +0600
Subject: [PATCH 1382/2145] [iconosquare] Extract comments

---
 youtube_dl/extractor/iconosquare.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index bb69c7a0f..a39f422e9 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -62,6 +62,14 @@ class IconosquareIE(InfoExtractor):
             'height': int_or_none(t.get('height'))
         } for thumbnail_id, t in media.get('images', {}).items()]
 
+        comments = [{
+            'id': comment.get('id'),
+            'text': comment['text'],
+            'timestamp': int_or_none(comment.get('created_time')),
+            'author': comment.get('from', {}).get('full_name'),
+            'author_id': comment.get('from', {}).get('username'),
+        } for comment in media.get('comments', {}).get('data', []) if 'text' in comment]
+
         return {
             'id': video_id,
             'title': title,
@@ -73,4 +81,5 @@ class IconosquareIE(InfoExtractor):
             'comment_count': comment_count,
             'like_count': like_count,
             'formats': formats,
+            'comments': comments,
         }

From 882fc9052e310b5ac6675488bba767c43ca2185e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 17:48:39 +0100
Subject: [PATCH 1383/2145] [condenast] fix extraction and add support for
 other sites

---
 youtube_dl/extractor/condenast.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 3db4db4e4..22c66da26 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -24,13 +23,25 @@ class CondeNastIE(InfoExtractor):
     # The keys are the supported sites and the values are the name to be shown
     # to the user and in the extractor description.
     _SITES = {
-        'wired': 'WIRED',
-        'gq': 'GQ',
-        'vogue': 'Vogue',
-        'glamour': 'Glamour',
-        'wmagazine': 'W Magazine',
-        'vanityfair': 'Vanity Fair',
+        'allure': 'Allure',
+        'architecturaldigest': 'Architectural Digest',
+        'arstechnica': 'Ars Technica',
+        'bonappetit': 'Bon Appetit',
+        'brides': 'Brides',
         'cnevids': 'Condé Nast',
+        'cntraveler': 'Condé Nast Traveler',
+        'details': 'Details',
+        'epicurious': 'Epicurious',
+        'glamour': 'Glamour',
+        'golfdigest': 'Golf Digest',
+        'gq': 'GQ',
+        'newyorker': 'The New Yorker',
+        'self': 'SELF',
+        'teenvogue': 'Teen Vogue',
+        'vanityfair': 'Vanity Fair',
+        'vogue': 'Vogue',
+        'wired': 'WIRED',
+        'wmagazine': 'W Magazine',
     }
 
     _VALID_URL = r'http://(video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
@@ -86,8 +97,8 @@ class CondeNastIE(InfoExtractor):
         info_url = base_info_url + data
         info_page = self._download_webpage(info_url, video_id,
                                            'Downloading video info')
-        video_info = self._search_regex(r'var video = ({.+?});', info_page, 'video info')
-        video_info = json.loads(video_info)
+        video_info = self._search_regex(r'var\s*video\s*=\s*({.+?});', info_page, 'video info')
+        video_info = self._parse_json(video_info, video_id)
 
         formats = [{
             'format_id': '%s-%s' % (fdata['type'].split('/')[-1], fdata['quality']),

From 2949a6cda9c39b3ff32891bdf0d6b48c46973f82 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 15:54:23 +0100
Subject: [PATCH 1384/2145] [condenast] fix video info regex

---
 youtube_dl/extractor/condenast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 22c66da26..d1380ac8d 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -97,7 +97,7 @@ class CondeNastIE(InfoExtractor):
         info_url = base_info_url + data
         info_page = self._download_webpage(info_url, video_id,
                                            'Downloading video info')
-        video_info = self._search_regex(r'var\s*video\s*=\s*({.+?});', info_page, 'video info')
+        video_info = self._search_regex(r'var\s+video\s*=\s*({.+?});', info_page, 'video info')
         video_info = self._parse_json(video_info, video_id)
 
         formats = [{

From 42ca72dff38c6cb23724dd91b39550e805bd8d25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 05:15:21 +0600
Subject: [PATCH 1385/2145] [condenast] Keep acute accent

---
 youtube_dl/extractor/condenast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index d1380ac8d..ef1d28091 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -26,7 +26,7 @@ class CondeNastIE(InfoExtractor):
         'allure': 'Allure',
         'architecturaldigest': 'Architectural Digest',
         'arstechnica': 'Ars Technica',
-        'bonappetit': 'Bon Appetit',
+        'bonappetit': 'Bon Appétit',
         'brides': 'Brides',
         'cnevids': 'Condé Nast',
         'cntraveler': 'Condé Nast Traveler',

From 0940c5b4c6a068d4919fd29a8db2a85ab3bbf703 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 05:18:45 +0600
Subject: [PATCH 1386/2145] [condenast] Do not capture unused group in
 _VALID_URL

---
 youtube_dl/extractor/condenast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index ef1d28091..d6949ca28 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -44,7 +44,7 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'http://(video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
     EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed)/.+?' % '|'.join(_SITES.keys())

From 1f9fb20fcda76f165ce39b01fe907fc74c8054d3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 07:39:22 +0100
Subject: [PATCH 1387/2145] [nextmedia] update AppleDailyIE tests

---
 youtube_dl/extractor/nextmedia.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c10784f6b..d1688457f 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -126,7 +126,8 @@ class AppleDailyIE(NextMediaIE):
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
             'upload_date': '20150128',
-        }
+        },
+        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
     }, {
         # No thumbnail
         'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
@@ -140,10 +141,19 @@ class AppleDailyIE(NextMediaIE):
         },
         'expected_warnings': [
             'video thumbnail',
-        ]
+        ],
+        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
     }, {
         'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
-        'only_matching': True,
+        'md5': 'eaa20e6b9df418c912d7f5dec2ba734d',
+        'info_dict': {
+            'id': '35770334',
+            'ext': 'mp4',
+            'title': '咖啡占卜測 XU裝熟指數',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:7b859991a6a4fedbdf3dd3b66545c748',
+            'upload_date': '20140417',
+        },
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From 2e40a12225fbe64b84b3975b3063a676df0f4522 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:24:35 +0800
Subject: [PATCH 1388/2145] [fktv] Correct spellings

---
 youtube_dl/extractor/fktv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index c2aa23aa2..93c4fd641 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -28,9 +28,9 @@ class FKTVIE(InfoExtractor):
 
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
-        matchs = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
-        if matchs:
-            poster, sources = matchs.groups()
+        matches = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
+        if matches:
+            poster, sources = matches.groups()
             urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
             if sources:
                 formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]

From 8ddf48d59f2d04b7411202eb6bf02c6eaa387035 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:48:51 +0800
Subject: [PATCH 1389/2145] [fktv] Raise an error is no videos found

---
 youtube_dl/extractor/fktv.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 93c4fd641..74c6cf866 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     clean_html,
     determine_ext,
+    ExtractorError,
 )
 
 
@@ -29,14 +30,15 @@ class FKTVIE(InfoExtractor):
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
         matches = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
-        if matches:
-            poster, sources = matches.groups()
-            urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
-            if sources:
-                formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
-                return {
-                    'id': episode,
-                    'title': title,
-                    'formats': formats,
-                    'thumbnail': poster,
-                }
+        if matches is None:
+            raise ExtractorError('Unable to extract the video')
+
+        poster, sources = matches.groups()
+        urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
+        formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
+        return {
+            'id': episode,
+            'title': title,
+            'formats': formats,
+            'thumbnail': poster,
+        }

From 140359fc2cfe7e9cbfecfd2fd625c6407232fe0f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:51:48 +0800
Subject: [PATCH 1390/2145] [fktv] Correct and improve some regexs

---
 youtube_dl/extractor/fktv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 74c6cf866..b081eb535 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -28,13 +28,13 @@ class FKTVIE(InfoExtractor):
         episode = self._match_id(url)
 
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
-        matches = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
+        title = clean_html(self._html_search_regex('<h3>([^<]+)</h3>', webpage, 'title'))
+        matches = re.search(r'(?s)<video[^>]+poster="([^"]+)"[^>]*>(.*)</video>', webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 
         poster, sources = matches.groups()
-        urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
+        urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
         return {
             'id': episode,

From 577380396171ba096240bfb3101f8151e32b587a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:58:44 +0800
Subject: [PATCH 1391/2145] [fktv] Correct thumbnail extraction and add the
 test

---
 youtube_dl/extractor/fktv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index b081eb535..289cbb8c8 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -21,6 +21,7 @@ class FKTVIE(InfoExtractor):
             'id': '1',
             'ext': 'mp4',
             'title': 'Folge 1 vom 10. April 2007',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
 
@@ -29,11 +30,14 @@ class FKTVIE(InfoExtractor):
 
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex('<h3>([^<]+)</h3>', webpage, 'title'))
-        matches = re.search(r'(?s)<video[^>]+poster="([^"]+)"[^>]*>(.*)</video>', webpage)
+        matches = re.search(r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 
         poster, sources = matches.groups()
+        if poster is None:
+            self.report_warning('unable to extract thumbnail')
+
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
         return {

From 711762f0b70b65f2f28f2cd8023497f50fccd81a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 18:01:08 +0800
Subject: [PATCH 1392/2145] [fktv] Coding style

---
 youtube_dl/extractor/fktv.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 289cbb8c8..0c14834f9 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -28,9 +28,12 @@ class FKTVIE(InfoExtractor):
     def _real_extract(self, url):
         episode = self._match_id(url)
 
-        webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex('<h3>([^<]+)</h3>', webpage, 'title'))
-        matches = re.search(r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
+        webpage = self._download_webpage(
+            'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
+        title = clean_html(self._html_search_regex(
+            '<h3>([^<]+)</h3>', webpage, 'title'))
+        matches = re.search(
+            r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 
@@ -39,7 +42,10 @@ class FKTVIE(InfoExtractor):
             self.report_warning('unable to extract thumbnail')
 
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
-        formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
+        formats = [{
+            'url': url,
+            'format_id': determine_ext(url),
+        } for url in urls]
         return {
             'id': episode,
             'title': title,

From 8de28761c41ca0451e3bbe4b9ee236d6651cca44 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 18:17:48 +0800
Subject: [PATCH 1393/2145] [fktv] Fix a regex

---
 youtube_dl/extractor/fktv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 0c14834f9..d9fc9952d 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -33,7 +33,8 @@ class FKTVIE(InfoExtractor):
         title = clean_html(self._html_search_regex(
             '<h3>([^<]+)</h3>', webpage, 'title'))
         matches = re.search(
-            r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
+            r'(?s)<video(?:(?!poster)[^>])+(?:poster="([^"]+)")?[^>]*>(.*)</video>',
+            webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 

From c44c7895b8774fb819b0b664bfcf64a7ebeea4e8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 11:28:26 +0100
Subject: [PATCH 1394/2145] [kuwo] fix title extraction and update test

---
 youtube_dl/extractor/kuwo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index fa233377d..51137a982 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -57,6 +57,7 @@ class KuwoIE(KuwoBaseIE):
             'upload_date': '20080122',
             'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
         },
+        'skip': 'this song has been offline because of copyright issues',
     }, {
         'url': 'http://www.kuwo.cn/yinyue/6446136/',
         'info_dict': {
@@ -78,7 +79,7 @@ class KuwoIE(KuwoBaseIE):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
+            r'(?s)class="[^"]*title[^"]*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', fatal=False)

From 3d09aa4c82100649279d979f9910a8c84ba301ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 11:40:32 +0100
Subject: [PATCH 1395/2145] [kuwo] extract title inside element with class
 title exactly

---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 51137a982..a3c260838 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -79,7 +79,7 @@ class KuwoIE(KuwoBaseIE):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'(?s)class="[^"]*title[^"]*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
+            r'(?s)class="(?:[^" ]+ +)*title(?: +[^" ]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', fatal=False)

From 4866b72eb2bf5747cde4654e75b1c1be0d456456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 25 Sep 2015 21:58:45 +0200
Subject: [PATCH 1396/2145] [fktv] Don't redefine 'url' in list comprehension

Detected with flake8.
---
 youtube_dl/extractor/fktv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index d9fc9952d..fb9739b49 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -44,9 +44,9 @@ class FKTVIE(InfoExtractor):
 
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{
-            'url': url,
+            'url': furl,
             'format_id': determine_ext(url),
-        } for url in urls]
+        } for furl in urls]
         return {
             'id': episode,
             'title': title,

From 4c917d0314f0442fd17d6f8ec8e583252167512c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 25 Sep 2015 22:02:48 +0200
Subject: [PATCH 1397/2145] [README.md] Document the 'duration' field in the
 output template (#6929)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 2ed751791..80152071d 100644
--- a/README.md
+++ b/README.md
@@ -281,6 +281,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video.
  - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
+ - `duration`: The sequence will be replaced by the length of the video in seconds.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 

From 08bea4adde5d5beb5f2a5cc875216b4205b90d50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 25 Sep 2015 22:34:02 +0200
Subject: [PATCH 1398/2145] Also run tests with python 3.5

---
 .travis.yml | 1 +
 tox.ini     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e78a2fa76..cc21fae8f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ python:
   - "3.2"
   - "3.3"
   - "3.4"
+  - "3.5"
 sudo: false
 script: nosetests test --verbose
 notifications:
diff --git a/tox.ini b/tox.ini
index cd805fe8a..48504329f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py33,py34
+envlist = py26,py27,py33,py34,py35
 [testenv]
 deps =
    nose

From 3706fb5dc8a38163689a29c7ed2b64f3b3d093e3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 07:51:11 +0100
Subject: [PATCH 1399/2145] [fktv] get format_id from video file ext

---
 youtube_dl/extractor/fktv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index fb9739b49..40ea27895 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -45,7 +45,7 @@ class FKTVIE(InfoExtractor):
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{
             'url': furl,
-            'format_id': determine_ext(url),
+            'format_id': determine_ext(furl),
         } for furl in urls]
         return {
             'id': episode,

From 5db34f680f93917ef79ba59d501b9f82e6d44330 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 10:31:32 +0100
Subject: [PATCH 1400/2145] [kuwo] check for the offline error page

---
 youtube_dl/extractor/kuwo.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a3c260838..36cb265ab 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -77,9 +77,11 @@ class KuwoIE(KuwoBaseIE):
         webpage = self._download_webpage(
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
+        if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
+            raise ExtractorError('this song has been offline because of copyright issues')
 
         song_name = self._html_search_regex(
-            r'(?s)class="(?:[^" ]+ +)*title(?: +[^" ]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
+            r'(?s)class="(?:[^"\s]+\s+)*title(?:\s+[^"\s]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', fatal=False)

From 7193650641f3fd84872b231e48263752d10b37b7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 11:44:35 +0100
Subject: [PATCH 1401/2145] [kuwo] treat the offline error as an expected
 ExtractorError

---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 36cb265ab..0c8ed5d07 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -78,7 +78,7 @@ class KuwoIE(KuwoBaseIE):
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
         if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
-            raise ExtractorError('this song has been offline because of copyright issues')
+            raise ExtractorError('this song has been offline because of copyright issues', expected=True)
 
         song_name = self._html_search_regex(
             r'(?s)class="(?:[^"\s]+\s+)*title(?:\s+[^"\s]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')

From fc42bc6ec9ab4858ec6bf1c8fe348ec47126375a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:45:43 +0600
Subject: [PATCH 1402/2145] [mtv] Look for sm4:video:embed (Closes #6936,
 closes #6970)

---
 youtube_dl/extractor/mtv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index a597714e9..4020cc2a9 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -200,7 +200,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mgid is None or ':' not in mgid:
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
-                webpage, 'mgid')
+                webpage, 'mgid', default=None)
+
+        if not mgid:
+            sm4_embed = self._html_search_meta(
+                'sm4:video:embed', webpage, 'sm4 embed', default='')
+            mgid = self._search_regex(
+                r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid')
 
         videos_info = self._get_videos_info(mgid)
         return videos_info

From fe1d858e35dde4c3068b9bec8a1bac7bd2cafadb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:46:42 +0600
Subject: [PATCH 1403/2145] [mtvservices:embedded] Add _extract_url

---
 youtube_dl/extractor/mtv.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 4020cc2a9..302c9bf35 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -228,6 +228,13 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
         },
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media.mtvnservices.com/embed/.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')

From 46fde8a1a2e72bca3c22b6de4db06490fb90b59a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:47:20 +0600
Subject: [PATCH 1404/2145] [extractor/generic] Use _extract_url for
 mtvservices

---
 youtube_dl/extractor/generic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8881a8a23..22ec78a02 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -50,6 +50,7 @@ from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
+from .mtv import MTVServicesEmbeddedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1611,12 +1612,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(url, ie='Vulture')
 
         # Look for embedded mtvservices player
-        mobj = re.search(
-            r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
-            webpage)
-        if mobj is not None:
-            url = unescapeHTML(mobj.group('url'))
-            return self.url_result(url, ie='MTVServicesEmbedded')
+        mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
+        if mtvservices_url:
+            return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
 
         # Look for embedded yahoo player
         mobj = re.search(

From ee3d5a6d476ccca626bd4878e2cf871373fe5ca9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:57:17 +0600
Subject: [PATCH 1405/2145] [bbc] Skip mediaselection on gelocation error
 (Closes #6983)

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 42526357a..c1692b6c5 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -294,7 +294,7 @@ class BBCCoUkIE(InfoExtractor):
                 return self._download_media_selector_url(
                     mediaselector_url % programme_id, programme_id)
             except BBCCoUkIE.MediaSelectionError as e:
-                if e.id == 'notukerror':
+                if e.id in ('notukerror', 'geolocation'):
                     last_exception = e
                     continue
                 self._raise_extractor_error(e)

From 26ccc68bedb3f35fcbaad0196da0a6d93eef6e89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:06:21 +0600
Subject: [PATCH 1406/2145] [bbc] Clarify iptv-all mediaset rationale

---
 youtube_dl/extractor/bbc.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c1692b6c5..074b56372 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -21,6 +21,9 @@ class BBCCoUkIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _MEDIASELECTOR_URLS = [
+        # Provides HQ HLS streams with even better quality that pc mediaset but fails
+        # with geolocation in some cases when it's even not geo restricted at all (e.g.
+        # http://www.bbc.co.uk/programmes/b06bp7lf)
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]

From 54914380c04ac7b52fa6dc69d49576fb7afe5f27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:07:12 +0600
Subject: [PATCH 1407/2145] [bbc] Add test for programme that fails with
 iptv-all mediaset

---
 youtube_dl/extractor/bbc.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 074b56372..cc2f6fed2 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -156,6 +156,21 @@ class BBCCoUkIE(InfoExtractor):
                 'skip_download': True,
             },
             'skip': 'geolocation',
+        }, {
+            # iptv-all mediaset fails with geolocation however there is no geo restriction
+            # for this programme at all
+            'url': 'http://www.bbc.co.uk/programmes/b06bp7lf',
+            'info_dict': {
+                'id': 'b06bp7kf',
+                'ext': 'flv',
+                'title': "Annie Mac's Friday Night, B.Traits sits in for Annie",
+                'description': 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.',
+                'duration': 10800,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,

From b65e5bb72f48c57316ac0166141810bd1e7b2de8 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Mon, 24 Aug 2015 00:31:30 +0800
Subject: [PATCH 1408/2145] [qqmusic] Add subtitles for QQMusic

Use .lrc lyrics as subtitles if lyrics in lrc format exist.
---
 youtube_dl/extractor/qqmusic.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 1654a641f..8ead04606 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -58,6 +58,11 @@ class QQMusicIE(InfoExtractor):
         curMs = int(time.time() * 1000) % 1000
         return int(round(random.random() * 2147483647) * curMs % 1E10)
 
+    def _filter_lrc(self, data):
+        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])'
+        texts = re.findall(lyrics_expr, data)
+        return ''.join(i + "\n" for i in texts)
+
     def _real_extract(self, url):
         mid = self._match_id(url)
 
@@ -112,15 +117,25 @@ class QQMusicIE(InfoExtractor):
         self._check_formats(formats, mid)
         self._sort_formats(formats)
 
-        return {
+        actual_lrc_lyrics = self._filter_lrc(lrc_content)
+
+        info_dict = {
             'id': mid,
             'formats': formats,
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,
             'description': lrc_content,
-            'thumbnail': thumbnail_url,
+            'thumbnail': thumbnail_url
         }
+        if actual_lrc_lyrics:
+            info_dict['subtitles'] = {
+                'origin': [{
+                    'ext': 'lrc',
+                    'data': actual_lrc_lyrics,
+                }]
+            }
+        return info_dict
 
 
 class QQPlaylistBaseIE(InfoExtractor):

From 961c5cbf172da49b26c09f806b0871719578e0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:38:11 +0600
Subject: [PATCH 1409/2145] [qqmusic] Eliminate _filter_lrc and use single
 quotes

---
 youtube_dl/extractor/qqmusic.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 8ead04606..bf6762034 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -58,11 +58,6 @@ class QQMusicIE(InfoExtractor):
         curMs = int(time.time() * 1000) % 1000
         return int(round(random.random() * 2147483647) * curMs % 1E10)
 
-    def _filter_lrc(self, data):
-        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])'
-        texts = re.findall(lyrics_expr, data)
-        return ''.join(i + "\n" for i in texts)
-
     def _real_extract(self, url):
         mid = self._match_id(url)
 
@@ -117,7 +112,9 @@ class QQMusicIE(InfoExtractor):
         self._check_formats(formats, mid)
         self._sort_formats(formats)
 
-        actual_lrc_lyrics = self._filter_lrc(lrc_content)
+        actual_lrc_lyrics = ''.join(
+            line + '\n' for line in re.findall(
+                r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
 
         info_dict = {
             'id': mid,

From d4cd06138c70dd50d574c9a5d96bf585648373d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:54:41 +0600
Subject: [PATCH 1410/2145] [qqmusic] Do not capture braced text from the
 middle of the string

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bf6762034..21cf6f2d6 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -114,7 +114,7 @@ class QQMusicIE(InfoExtractor):
 
         actual_lrc_lyrics = ''.join(
             line + '\n' for line in re.findall(
-                r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
+                r'(?m)^(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
 
         info_dict = {
             'id': mid,

From cd1bb54990dbe97b74a184022403cc30b0dbfe05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:00:59 +0600
Subject: [PATCH 1411/2145] [qqmusic] Add test for a song with non .lrc lyrics

---
 youtube_dl/extractor/qqmusic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 21cf6f2d6..50b0d9f31 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -43,6 +43,21 @@ class QQMusicIE(InfoExtractor):
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
+    }, {
+        'note': 'lyrics not in .lrc format',
+        'url': 'http://y.qq.com/#type=song&mid=001JyApY11tIp6',
+        'info_dict': {
+            'id': '001JyApY11tIp6',
+            'ext': 'mp3',
+            'title': 'Shadows Over Transylvania',
+            'upload_date': '19970225',
+            'creator': 'Dark Funeral',
+            'description': 'md5:ed14d5bd7ecec19609108052c25b2c11',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _FORMATS = {

From 26c6d1922eaf70c980a054f0523c112f71863465 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:01:28 +0600
Subject: [PATCH 1412/2145] Credit @fqj1994 for qqmusic .lrc support

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 901c1b263..cc552bcb2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -143,3 +143,4 @@ Shaun Walbridge
 Lee Jenkins
 Anssi Hannula
 Lukáš Lalinský
+Qijiang Fan

From 8aab976bbd0a009a15b1737668882e44d84c7a71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:07:54 +0600
Subject: [PATCH 1413/2145] [extractor/common] Document release_date field

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1e7db8a9b..2fe0d5d37 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -152,6 +152,7 @@ class InfoExtractor(object):
     description:    Full video description.
     uploader:       Full name of the video uploader.
     creator:        The main artist who created the video.
+    release_date:   The date (YYYYMMDD) when the video was released.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.

From a5d09d684eef68b42377a651706a61b0b3fca329 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:08:23 +0600
Subject: [PATCH 1414/2145] [qqmusic] Use release_date

---
 youtube_dl/extractor/qqmusic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 50b0d9f31..c88823416 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -25,7 +25,7 @@ class QQMusicIE(InfoExtractor):
             'id': '004295Et37taLD',
             'ext': 'mp3',
             'title': '可惜没如果',
-            'upload_date': '20141227',
+            'release_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -38,7 +38,7 @@ class QQMusicIE(InfoExtractor):
             'id': '004MsGEo3DdNxV',
             'ext': 'mp3',
             'title': '如果',
-            'upload_date': '20050626',
+            'release_date': '20050626',
             'creator': '李季美',
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -50,7 +50,7 @@ class QQMusicIE(InfoExtractor):
             'id': '001JyApY11tIp6',
             'ext': 'mp3',
             'title': 'Shadows Over Transylvania',
-            'upload_date': '19970225',
+            'release_date': '19970225',
             'creator': 'Dark Funeral',
             'description': 'md5:ed14d5bd7ecec19609108052c25b2c11',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -135,7 +135,7 @@ class QQMusicIE(InfoExtractor):
             'id': mid,
             'formats': formats,
             'title': song_name,
-            'upload_date': publish_time,
+            'release_date': publish_time,
             'creator': singer,
             'description': lrc_content,
             'thumbnail': thumbnail_url

From 95c5e101035eb3a77ebf62d75b2ebaac836373c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:15:34 +0600
Subject: [PATCH 1415/2145] [qqmusic] Allow [mm:ss] timestamps

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c88823416..c98539f6a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -129,7 +129,7 @@ class QQMusicIE(InfoExtractor):
 
         actual_lrc_lyrics = ''.join(
             line + '\n' for line in re.findall(
-                r'(?m)^(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
+                r'(?m)^(\[[0-9]{2}:[0-9]{2}(?:\.[0-9]{2,})?\][^\n]*|\[[^\]]*\])', lrc_content))
 
         info_dict = {
             'id': mid,

From 9d632b1b279ca9bc5b73011b942c7fb452370788 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 15:30:02 +0100
Subject: [PATCH 1416/2145] [eagleplatform] extract mp4 url and fix thumbnail
 url

---
 youtube_dl/extractor/eagleplatform.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index a1ee51568..2987d5272 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -21,7 +21,7 @@ class EaglePlatformIE(InfoExtractor):
     _TESTS = [{
         # http://lenta.ru/news/2015/03/06/navalny/
         'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
-        'md5': '0b7994faa2bd5c0f69a3db6db28d078d',
+        'md5': '70f5187fb620f2c1d503b3b22fd4efe3',
         'info_dict': {
             'id': '227304',
             'ext': 'mp4',
@@ -36,7 +36,7 @@ class EaglePlatformIE(InfoExtractor):
         # http://muz-tv.ru/play/7129/
         # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
         'url': 'eagleplatform:media.clipyou.ru:12820',
-        'md5': '6c2ebeab03b739597ce8d86339d5a905',
+        'md5': '90b26344ba442c8e44aa4cf8f301164a',
         'info_dict': {
             'id': '12820',
             'ext': 'mp4',
@@ -53,10 +53,10 @@ class EaglePlatformIE(InfoExtractor):
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
-        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        response = self._download_json(url_or_request, video_id, note)
         self._handle_error(response)
-        return response
+        return response['data'][0]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -69,7 +69,7 @@ class EaglePlatformIE(InfoExtractor):
 
         title = media['title']
         description = media.get('description')
-        thumbnail = media.get('snapshot')
+        thumbnail = self._proto_relative_url(media.get('snapshot'))
         duration = int_or_none(media.get('duration'))
         view_count = int_or_none(media.get('views'))
 
@@ -78,13 +78,18 @@ class EaglePlatformIE(InfoExtractor):
         if age_restriction:
             age_limit = 0 if age_restriction == 'allow_all' else 18
 
-        m3u8_data = self._download_json(
-            self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:'),
-            video_id, 'Downloading m3u8 JSON')
+        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'])
 
+        m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
         formats = self._extract_m3u8_formats(
-            m3u8_data['data'][0], video_id,
+            m3u8_url, video_id,
             'mp4', entry_protocol='m3u8_native')
+
+        mp4_url = self._get_video_url(
+            secure_m3u8.replace("m3u8", "mp4").replace("hlsvod", "mp4").replace("hls", "mp4"),
+            video_id, 'Downloading mp4 JSON')
+        formats.append({'url': mp4_url, 'format_id': 'mp4'})
+
         self._sort_formats(formats)
 
         return {

From 22becac4bd4338108b7949607fb7dccd6d173abf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 17:37:30 +0100
Subject: [PATCH 1417/2145] [eagleplatform] return the code to handle errors in
 all _download_json requests

---
 youtube_dl/extractor/eagleplatform.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 2987d5272..75cf73b25 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -53,10 +53,13 @@ class EaglePlatformIE(InfoExtractor):
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
-    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
-        response = self._download_json(url_or_request, video_id, note)
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
         self._handle_error(response)
-        return response['data'][0]
+        return response
+
+    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        return self._download_json(url_or_request, video_id, note)['data'][0]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From d045f0bdb72bc3f84cbcfa5fd940c46d3360b386 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 18:53:57 +0100
Subject: [PATCH 1418/2145] [eagleplatform] use http urls explicitly

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 75cf73b25..8bf88c77d 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -81,7 +81,7 @@ class EaglePlatformIE(InfoExtractor):
         if age_restriction:
             age_limit = 0 if age_restriction == 'allow_all' else 18
 
-        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'])
+        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:')
 
         m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
         formats = self._extract_m3u8_formats(

From c471b34575eac87b0841616dd6acb90b221c6fe6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 01:10:39 +0600
Subject: [PATCH 1419/2145] [eagleplatform] Simplify secure mp4 construction
 and clarify rationale

---
 youtube_dl/extractor/eagleplatform.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 8bf88c77d..43336e57f 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -89,7 +89,9 @@ class EaglePlatformIE(InfoExtractor):
             'mp4', entry_protocol='m3u8_native')
 
         mp4_url = self._get_video_url(
-            secure_m3u8.replace("m3u8", "mp4").replace("hlsvod", "mp4").replace("hls", "mp4"),
+            # Secure mp4 URL is constructed according to Player.prototype.mp4 from
+            # http://lentaru.media.eagleplatform.com/player/player.js
+            re.sub(r'm3u8|hlsvod|hls|f4m', 'mp4', secure_m3u8),
             video_id, 'Downloading mp4 JSON')
         formats.append({'url': mp4_url, 'format_id': 'mp4'})
 

From 3c63e1bb5740484dae483595a87325e8cf9c7e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 01:12:46 +0600
Subject: [PATCH 1420/2145] [eagleplatform] Make _handle_error staticmethod

---
 youtube_dl/extractor/eagleplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 43336e57f..ee24242a8 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -48,7 +48,8 @@ class EaglePlatformIE(InfoExtractor):
         'skip': 'Georestricted',
     }]
 
-    def _handle_error(self, response):
+    @staticmethod
+    def _handle_error(response):
         status = int_or_none(response.get('status', 200))
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)

From 2f962d0a91d22becd59489a9d879b1caafadefed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 01:17:44 +0600
Subject: [PATCH 1421/2145] [eagleplatform] Use http scheme for thumbnail

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index ee24242a8..e529b9b96 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -73,7 +73,7 @@ class EaglePlatformIE(InfoExtractor):
 
         title = media['title']
         description = media.get('description')
-        thumbnail = self._proto_relative_url(media.get('snapshot'))
+        thumbnail = self._proto_relative_url(media.get('snapshot'), 'http:')
         duration = int_or_none(media.get('duration'))
         view_count = int_or_none(media.get('views'))
 

From 95240b8093c6fe9007e33f0991ed8d54b42eaf3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 15 Sep 2015 21:30:24 +0800
Subject: [PATCH 1422/2145] Use `insert` for all sys.path manipulations

Closes #6867.
---
 devscripts/bash-completion.py       | 2 +-
 devscripts/fish-completion.py       | 2 +-
 devscripts/gh-pages/update-sites.py | 2 +-
 devscripts/make_supportedsites.py   | 2 +-
 devscripts/zsh-completion.py        | 2 +-
 youtube_dl/__main__.py              | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index cd26cc089..ce68f26f9 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -5,7 +5,7 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 
 BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index c2f238798..41629d87d 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -6,7 +6,7 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 from youtube_dl.utils import shell_quote
 
diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index d3ef5f0b5..503c1372f 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -6,7 +6,7 @@ import os
 import textwrap
 
 # We must be able to import youtube_dl
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 
 import youtube_dl
 
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index 3df4385a6..8cb4a4638 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -9,7 +9,7 @@ import sys
 
 # Import youtube_dl
 ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
-sys.path.append(ROOT_DIR)
+sys.path.insert(0, ROOT_DIR)
 import youtube_dl
 
 
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
index f200f2c80..04728e8e2 100755
--- a/devscripts/zsh-completion.py
+++ b/devscripts/zsh-completion.py
@@ -5,7 +5,7 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 
 ZSH_COMPLETION_FILE = "youtube-dl.zsh"
diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py
index 65a0f891c..42a0f8c6f 100755
--- a/youtube_dl/__main__.py
+++ b/youtube_dl/__main__.py
@@ -11,7 +11,7 @@ if __package__ is None and not hasattr(sys, "frozen"):
     # direct call of __main__.py
     import os.path
     path = os.path.realpath(os.path.abspath(__file__))
-    sys.path.append(os.path.dirname(os.path.dirname(path)))
+    sys.path.insert(0, os.path.dirname(os.path.dirname(path)))
 
 import youtube_dl
 

From c6b8f4d0c9413df9c33f3bd67859d1a9675ec8aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 05:53:21 +0600
Subject: [PATCH 1423/2145] [condenast] Add support for JS embeds

---
 youtube_dl/extractor/condenast.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index d6949ca28..6f92ae2ed 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     orderedSet,
+    remove_end,
 )
 
 
@@ -44,12 +45,12 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
-    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed)/.+?' % '|'.join(_SITES.keys())
+    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
         'md5': '1921f713ed48aabd715691f774c451f7',
         'info_dict': {
@@ -58,7 +59,16 @@ class CondeNastIE(InfoExtractor):
             'title': '3D Printed Speakers Lit With LED',
             'description': 'Check out these beautiful 3D printed LED speakers.  You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.',
         }
-    }
+    }, {
+        # JS embed
+        'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',
+        'md5': 'f1a6f9cafb7083bab74a710f65d08999',
+        'info_dict': {
+            'id': '55f9cf8b61646d1acf00000c',
+            'ext': 'mp4',
+            'title': '3D printed TSA Travel Sentry keys really do open TSA locks',
+        }
+    }]
 
     def _extract_series(self, url, webpage):
         title = self._html_search_regex(r'<div class="cne-series-info">.*?<h1>(.+?)</h1>',
@@ -122,6 +132,13 @@ class CondeNastIE(InfoExtractor):
         url_type = mobj.group('type')
         item_id = mobj.group('id')
 
+        # Convert JS embed to regular embed
+        if url_type == 'embedjs':
+            parsed_url = compat_urlparse.urlparse(url)
+            url = compat_urlparse.urlunparse(parsed_url._replace(
+                path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
+            url_type = 'embed'
+
         self.to_screen('Extracting from %s with the Condé Nast extractor' % self._SITES[site])
         webpage = self._download_webpage(url, item_id)
 

From dd467d33d0883c0c0c94d7fc5ed041d5c12bb564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 05:55:48 +0600
Subject: [PATCH 1424/2145] [extractor/generic] Add support for condenast
 script embeds (Closes #6885, closes #6991)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 22ec78a02..ca5fbafb2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1653,7 +1653,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'MLB')
 
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
+            r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
             webpage)
         if mobj is not None:
             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')

From e13b9e7885d83f09d95882a494d5e32153e7baa6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 22:38:00 +0100
Subject: [PATCH 1425/2145] [keek] fix info extraction

---
 youtube_dl/extractor/keek.py | 28 ++++++----------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index c0956ba09..4493a929d 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -4,43 +4,27 @@ from .common import InfoExtractor
 
 
 class KeekIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?keek\.com/keek/(?P<id>\w+)'
     IE_NAME = 'keek'
     _TEST = {
-        'url': 'https://www.keek.com/ytdl/keeks/NODfbab',
-        'md5': '09c5c109067536c1cec8bac8c21fea05',
+        'url': 'https://www.keek.com/keek/NODfbab',
+        'md5': '9b0636f8c0f7614afa4ea5e4c6e57e83',
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'uploader': 'youtube-dl project',
-            'uploader_id': 'ytdl',
-            'title': 'test chars: "\'/\\\u00e4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
+            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_url = 'http://cdn.keek.com/keek/video/%s' % video_id
-        thumbnail = 'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
         webpage = self._download_webpage(url, video_id)
 
-        raw_desc = self._html_search_meta('description', webpage)
-        if raw_desc:
-            uploader = self._html_search_regex(
-                r'Watch (.*?)\s+\(', raw_desc, 'uploader', fatal=False)
-            uploader_id = self._html_search_regex(
-                r'Watch .*?\(@(.+?)\)', raw_desc, 'uploader_id', fatal=False)
-        else:
-            uploader = None
-            uploader_id = None
-
         return {
             'id': video_id,
-            'url': video_url,
+            'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
             'title': self._og_search_title(webpage),
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From 6a24cb3d22876826477dc2857f9d46fe65a9fc28 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 18:04:25 +0100
Subject: [PATCH 1426/2145] [keek] extract more info

---
 youtube_dl/extractor/keek.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 4493a929d..24f5c7cb7 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
@@ -13,6 +15,9 @@ class KeekIE(InfoExtractor):
             'id': 'NODfbab',
             'ext': 'mp4',
             'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
+            'description': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
+            'uploader': 'ytdl',
+            'uploader_id': 'eGT5bab',
         },
     }
 
@@ -20,11 +25,18 @@ class KeekIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        uploader = uploader_id = None
+        matches = re.search(r'data-username="(?P<uploader>[^"]+)"[^>]*data-user-id="(?P<uploader_id>[^"]+)"', webpage)
+        if matches:
+            uploader, uploader_id = matches.groups()
 
         return {
             'id': video_id,
             'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
             'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
+            'uploader': uploader,
+            'uploader_id': uploader_id,
         }

From 80e98aed69102b0f300ac69001ab6ac64e2febec Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 18:25:24 +0100
Subject: [PATCH 1427/2145] [keek] fix test title

---
 youtube_dl/extractor/keek.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 24f5c7cb7..280134a2a 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -14,8 +14,8 @@ class KeekIE(InfoExtractor):
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
-            'description': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
+            'title': 'test chars: "\'/\\\xe4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
+            'description': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
             'uploader': 'ytdl',
             'uploader_id': 'eGT5bab',
         },

From f30e9976d62c528bc7eda7b90d7c2fbc89d0b0be Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 19:07:18 +0100
Subject: [PATCH 1428/2145] [keek] add utf-8 coding cookie

---
 youtube_dl/extractor/keek.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 280134a2a..d84774dca 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -14,7 +15,7 @@ class KeekIE(InfoExtractor):
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'title': 'test chars: "\'/\\\xe4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
+            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
             'description': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
             'uploader': 'ytdl',
             'uploader_id': 'eGT5bab',

From 98e1c935a1ed676ca747c04c28d98370f740cadc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Sep 2015 19:09:48 +0100
Subject: [PATCH 1429/2145] [keek] extract uploader and uploader id with
 _search_regex

---
 youtube_dl/extractor/keek.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index d84774dca..99154112b 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -26,10 +24,6 @@ class KeekIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        uploader = uploader_id = None
-        matches = re.search(r'data-username="(?P<uploader>[^"]+)"[^>]*data-user-id="(?P<uploader_id>[^"]+)"', webpage)
-        if matches:
-            uploader, uploader_id = matches.groups()
 
         return {
             'id': video_id,
@@ -38,6 +32,6 @@ class KeekIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': uploader,
-            'uploader_id': uploader_id,
+            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', None),
+            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', None),
         }

From 817690ff736c276835e16860957d758d92afb2fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:05:24 +0600
Subject: [PATCH 1430/2145] [keek] Make uploader fields non fatal

---
 youtube_dl/extractor/keek.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 99154112b..7e1a3f67c 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -32,6 +32,6 @@ class KeekIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', None),
-            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', None),
+            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', fatal=False),
+            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', fatal=False),
         }

From 926fb62eec016db9ef23e9d8d210c8fb2e4de20e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:14:14 +0600
Subject: [PATCH 1431/2145] [keek] Remove description

Since it equals title plus static suffix
---
 youtube_dl/extractor/keek.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 7e1a3f67c..dff4cb558 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -13,8 +13,7 @@ class KeekIE(InfoExtractor):
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
-            'description': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
+            'title': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
             'uploader': 'ytdl',
             'uploader_id': 'eGT5bab',
         },
@@ -29,8 +28,7 @@ class KeekIE(InfoExtractor):
             'id': video_id,
             'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
+            'title': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', fatal=False),
             'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', fatal=False),

From 1ab1c4ef5708fc1db50c5b875dcbcfe9ce984035 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:15:13 +0600
Subject: [PATCH 1432/2145] [keek] Improve uploader fields regexes

---
 youtube_dl/extractor/keek.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index dff4cb558..dfd800a91 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -30,6 +30,10 @@ class KeekIE(InfoExtractor):
             'ext': 'mp4',
             'title': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', fatal=False),
-            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', fatal=False),
+            'uploader': self._search_regex(
+                r'data-username=(["\'])(?P<uploader>.+?)\1', webpage,
+                'uploader', fatal=False, group='uploader'),
+            'uploader_id': self._search_regex(
+                r'data-user-id=(["\'])(?P<uploader_id>.+?)\1', webpage,
+                'uploader id', fatal=False, group='uploader_id'),
         }

From 86692c019cb2baeacfdf556ea6ca026dd4ac49e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:17:28 +0600
Subject: [PATCH 1433/2145] [keek] Strip title

---
 youtube_dl/extractor/keek.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index dfd800a91..94a03d277 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -28,7 +28,7 @@ class KeekIE(InfoExtractor):
             'id': video_id,
             'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
-            'title': self._og_search_description(webpage),
+            'title': self._og_search_description(webpage).strip(),
             'thumbnail': self._og_search_thumbnail(webpage),
             'uploader': self._search_regex(
                 r'data-username=(["\'])(?P<uploader>.+?)\1', webpage,

From c254f75bbb68c4d6fdb9e63bfa399d7eff7b2c23 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 28 Sep 2015 04:42:11 +0200
Subject: [PATCH 1434/2145] release 2015.09.28

---
 docs/supportedsites.md | 4 +---
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ab153af6b..fa83b68ad 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -101,7 +101,7 @@
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
- - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED
+ - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Cracked**
  - **Criterion**
  - **CrooksAndLiars**
@@ -158,7 +158,6 @@
  - **faz.net**
  - **fc2**
  - **fernsehkritik.tv**
- - **fernsehkritik.tv:postecke**
  - **Firstpost**
  - **FiveTV**
  - **Flickr**
@@ -208,7 +207,6 @@
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
- - **HostingBulk**
  - **HotNewHipHop**
  - **Howcast**
  - **HowStuffWorks**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ef4f2755..8f0977849 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.22'
+__version__ = '2015.09.28'

From 8a64969404ecbbac9cb8b970f7ca36a9e6c03cf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Sep 2015 21:33:21 +0600
Subject: [PATCH 1435/2145] [adultswim] Prefer stream (Closes #7015)

---
 youtube_dl/extractor/adultswim.py | 37 ++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 4327c2f61..27de07587 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     xpath_text,
@@ -123,7 +124,6 @@ class AdultSwimIE(InfoExtractor):
         else:
             collections = bootstrapped_data['show']['collections']
             collection, video_info = self.find_collection_containing_video(collections, episode_path)
-
             # Video wasn't found in the collections, let's try `slugged_video`.
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
@@ -133,7 +133,9 @@ class AdultSwimIE(InfoExtractor):
 
             show = bootstrapped_data['show']
             show_title = show['title']
-            segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+            stream = video_info.get('stream')
+            clips = [stream] if stream else video_info['clips']
+            segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']
         episode_title = video_info['title']
@@ -142,7 +144,7 @@ class AdultSwimIE(InfoExtractor):
 
         entries = []
         for part_num, segment_id in enumerate(segment_ids):
-            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=mobile' % segment_id
+            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id
 
             segment_title = '%s - %s' % (show_title, episode_title)
             if len(segment_ids) > 1:
@@ -158,17 +160,30 @@ class AdultSwimIE(InfoExtractor):
             formats = []
             file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')
 
+            unique_urls = []
+            unique_file_els = []
             for file_el in file_els:
+                media_url = file_el.text
+                if not media_url or determine_ext(media_url) == 'f4m':
+                    continue
+                if file_el.text not in unique_urls:
+                    unique_urls.append(file_el.text)
+                    unique_file_els.append(file_el)
+
+            for file_el in unique_file_els:
                 bitrate = file_el.attrib.get('bitrate')
                 ftype = file_el.attrib.get('type')
-
-                formats.append({
-                    'format_id': '%s_%s' % (bitrate, ftype),
-                    'url': file_el.text.strip(),
-                    # The bitrate may not be a number (for example: 'iphone')
-                    'tbr': int(bitrate) if bitrate.isdigit() else None,
-                    'quality': 1 if ftype == 'hd' else -1
-                })
+                media_url = file_el.text
+                if determine_ext(media_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        media_url, segment_title, 'mp4', 'm3u8_native', preference=0, m3u8_id='hls'))
+                else:
+                    formats.append({
+                        'format_id': '%s_%s' % (bitrate, ftype),
+                        'url': file_el.text.strip(),
+                        # The bitrate may not be a number (for example: 'iphone')
+                        'tbr': int(bitrate) if bitrate.isdigit() else None,
+                    })
 
             self._sort_formats(formats)
 

From c01e1a96aa964ef6d5f0bf7675dbe34096b1d2c8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Sep 2015 11:20:43 +0100
Subject: [PATCH 1436/2145] [brightcove] fix test and fields extraction

---
 youtube_dl/extractor/brightcove.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index e4a7befee..b41cee91b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -354,14 +354,18 @@ class BrightcoveIE(InfoExtractor):
 
 class BrightcoveInPageEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
-    TEST = {
+    _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
+        'md5': 'c8100925723840d4b0d243f7025703be',
         'info_dict': {
             'id': '4463358922001',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Meet the man behind Popcorn Time',
-            'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+            'description': 'md5:eac376a4fe366edc70279bfb681aea16',
+            'timestamp': 1441391203,
+            'upload_date': '20150904',
             'duration': 165768,
+            'uploader_id': '929656772001',
         }
     }
 
@@ -403,7 +407,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
 
         title = json_data['name']
         description = json_data.get('description')
-        thumbnail = json_data.get('name')
+        thumbnail = json_data.get('thumbnail')
         timestamp = parse_iso8601(json_data.get('published_at'))
         duration = int_or_none(json_data.get('duration'))
 
@@ -417,12 +421,13 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
                 if src:
                     formats.append({
                         'url': src,
-                        'abr': source.get('avg_bitrate'),
+                        'tbr': source.get('avg_bitrate'),
                         'width': int_or_none(source.get('width')),
                         'height': int_or_none(source.get('height')),
                         'filesize': source.get('size'),
                         'container': source.get('container'),
-                        'vcodec': source.get('container'),
+                        'vcodec': source.get('codec'),
+                        'ext': source.get('container').lower(),
                     })
 
         self._sort_formats(formats)
@@ -435,4 +440,5 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
+            'uploader_id': account_id,
         }

From c596ce91cd22914e031b9ac94ec38d1137e50d60 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Sep 2015 15:39:47 +0200
Subject: [PATCH 1437/2145] [comedycentral] Fix youtube-dl :thedailyshow

We'll let the generic IE follow the redirect and call back to us with the episode URL
---
 youtube_dl/extractor/comedycentral.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 91ebb0ce5..3e4bd10b6 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -151,12 +151,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         if mobj.group('shortname'):
-            if mobj.group('shortname') in ('tds', 'thedailyshow'):
-                url = 'http://thedailyshow.cc.com/full-episodes/'
-            else:
-                url = 'http://thecolbertreport.cc.com/full-episodes/'
-            mobj = re.match(self._VALID_URL, url, re.VERBOSE)
-            assert mobj is not None
+            return self.url_result('http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes')
 
         if mobj.group('clip'):
             if mobj.group('videotitle'):

From aedb930cfcd068600d3b5b0b3aef32df1a28f0c0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Sep 2015 13:30:48 +0100
Subject: [PATCH 1438/2145] [nfl] fix content id regex(fixes #7012)

---
 youtube_dl/extractor/nfl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 55dc6107d..4c35c561b 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -151,7 +151,7 @@ class NFLIE(InfoExtractor):
             group='config'))
         # For articles, the id in the url is not the video id
         video_id = self._search_regex(
-            r'(?:<nflcs:avplayer[^>]+data-contentId\s*=\s*|contentId\s*:\s*)(["\'])(?P<id>.+?)\1',
+            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>.+?)\1',
             webpage, 'video id', default=video_id, group='id')
         config = self._download_json(config_url, video_id, 'Downloading player config')
         url_template = NFLIE.prepend_host(

From ee2d190253ae3e0a8254029dc6e8d7e42c194042 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:06:21 +0600
Subject: [PATCH 1439/2145] [nfl] Add test for #7012

---
 youtube_dl/extractor/nfl.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 4c35c561b..200874d68 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -107,6 +107,20 @@ class NFLIE(InfoExtractor):
             'timestamp': 1442618809,
             'upload_date': '20150918',
         },
+    }, {
+        # lowercase data-contentid
+        'url': 'http://www.steelers.com/news/article-1/Tomlin-on-Ben-getting-Vick-ready/56399c96-4160-48cf-a7ad-1d17d4a3aef7',
+        'info_dict': {
+            'id': '12693586-6ea9-4743-9c1c-02c59e4a5ef2',
+            'ext': 'mp4',
+            'title': 'Tomlin looks ahead to Ravens on a short week',
+            'description': 'md5:32f3f7b139f43913181d5cbb24ecad75',
+            'timestamp': 1443459651,
+            'upload_date': '20150928',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
         'only_matching': True,

From 93bc7ef165731c729fd6fbbb3e95ecbbb16b4c21 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Sun, 30 Aug 2015 14:33:12 +0800
Subject: [PATCH 1440/2145] [test] recursively check dict and list in
 expect_info_dict

This allows to use md5:, re:, etc within the str inside a list
or dict.
---
 test/helper.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index cb6eec8d9..6612970ef 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,7 +89,7 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
-def expect_info_dict(self, got_dict, expected_dict):
+def expect_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
         if isinstance(expected, compat_str) and expected.startswith('re:'):
             got = got_dict.get(info_field)
@@ -127,6 +127,22 @@ def expect_info_dict(self, got_dict, expected_dict):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
                             'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
+        elif isinstance(expected, dict) and isinstance(got_dict.get(info_field, None), dict):
+            expect_dict(self, got_dict.get(info_field), expected)
+        elif isinstance(expected, list) and isinstance(got_dict.get(info_field, None), list):
+            got = got_dict.get(info_field, None)
+            self.assertEqual(len(expected), len(got),
+                             'Expect a list of length %d, but got a list of length %d' % (
+                             len(expected), len(got)))
+            _id = 0
+            for i, j in zip(got, expected):
+                _type_i = type(i)
+                _type_j = type(j)
+                self.assertEqual(_type_j, _type_i,
+                                 'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                                 _id, info_field, _type_j, _type_i))
+                expect_dict(self, {'_': i}, {'_': j})
+                _id += 1
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
@@ -149,6 +165,9 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertEqual(expected, got,
                              'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
+
+def expect_info_dict(self, got_dict, expected_dict):
+    expect_dict(self, got_dict, expected_dict)
     # Check for the presence of mandatory fields
     if got_dict.get('_type') not in ('playlist', 'multi_video'):
         for key in ('id', 'url', 'title', 'ext'):

From 40c931de4b3c06c1959362c2380f4bf243ef5702 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Sat, 26 Sep 2015 23:10:38 +0800
Subject: [PATCH 1441/2145] [test] split expect_dict to two functions

---
 test/helper.py | 143 ++++++++++++++++++++++++-------------------------
 1 file changed, 70 insertions(+), 73 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 6612970ef..e50d7ff28 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,81 +89,78 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
+def expect_value(self, got, expected, field):
+    if isinstance(expected, compat_str) and expected.startswith('re:'):
+        match_str = expected[len('re:'):]
+        match_rex = re.compile(match_str)
+
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            match_rex.match(got),
+            'field %s (value: %r) should match %r' % (field, got, match_str))
+    elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
+        start_str = expected[len('startswith:'):]
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            got.startswith(start_str),
+            'field %s (value: %r) should start with %r' % (field, got, start_str))
+    elif isinstance(expected, compat_str) and expected.startswith('contains:'):
+        contains_str = expected[len('contains:'):]
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            contains_str in got,
+            'field %s (value: %r) should contain %r' % (field, got, contains_str))
+    elif isinstance(expected, type):
+        self.assertTrue(isinstance(got, expected),
+                        'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+    elif isinstance(expected, dict) and isinstance(got, dict):
+        expect_dict(self, got, expected)
+    elif isinstance(expected, list) and isinstance(got, list):
+        self.assertEqual(len(expected), len(got),
+                         'Expect a list of length %d, but got a list of length %d' % (
+                         len(expected), len(got)))
+        _id = 0
+        for i, j in zip(got, expected):
+            _type_i = type(i)
+            _type_j = type(j)
+            self.assertEqual(_type_j, _type_i,
+                             'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                             _id, field, _type_j, _type_i))
+            expect_value(self, i, j, field)
+            _id += 1
+    else:
+        if isinstance(expected, compat_str) and expected.startswith('md5:'):
+            got = 'md5:' + md5(got)
+        elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+            self.assertTrue(
+                isinstance(got, (list, dict)),
+                'Expected field %s to be a list or a dict, but it is of type %s' % (
+                    field, type(got).__name__))
+            expected_num = int(expected.partition(':')[2])
+            assertGreaterEqual(
+                self, len(got), expected_num,
+                'Expected %d items in field %s, but only got %d' % (
+                    expected_num, field, len(got)
+                )
+            )
+            return
+        self.assertEqual(expected, got,
+                         'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+
+
 def expect_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
-        if isinstance(expected, compat_str) and expected.startswith('re:'):
-            got = got_dict.get(info_field)
-            match_str = expected[len('re:'):]
-            match_rex = re.compile(match_str)
-
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                match_rex.match(got),
-                'field %s (value: %r) should match %r' % (info_field, got, match_str))
-        elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
-            got = got_dict.get(info_field)
-            start_str = expected[len('startswith:'):]
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                got.startswith(start_str),
-                'field %s (value: %r) should start with %r' % (info_field, got, start_str))
-        elif isinstance(expected, compat_str) and expected.startswith('contains:'):
-            got = got_dict.get(info_field)
-            contains_str = expected[len('contains:'):]
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                contains_str in got,
-                'field %s (value: %r) should contain %r' % (info_field, got, contains_str))
-        elif isinstance(expected, type):
-            got = got_dict.get(info_field)
-            self.assertTrue(isinstance(got, expected),
-                            'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
-        elif isinstance(expected, dict) and isinstance(got_dict.get(info_field, None), dict):
-            expect_dict(self, got_dict.get(info_field), expected)
-        elif isinstance(expected, list) and isinstance(got_dict.get(info_field, None), list):
-            got = got_dict.get(info_field, None)
-            self.assertEqual(len(expected), len(got),
-                             'Expect a list of length %d, but got a list of length %d' % (
-                             len(expected), len(got)))
-            _id = 0
-            for i, j in zip(got, expected):
-                _type_i = type(i)
-                _type_j = type(j)
-                self.assertEqual(_type_j, _type_i,
-                                 'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                                 _id, info_field, _type_j, _type_i))
-                expect_dict(self, {'_': i}, {'_': j})
-                _id += 1
-        else:
-            if isinstance(expected, compat_str) and expected.startswith('md5:'):
-                got = 'md5:' + md5(got_dict.get(info_field))
-            elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
-                got = got_dict.get(info_field)
-                self.assertTrue(
-                    isinstance(got, (list, dict)),
-                    'Expected field %s to be a list or a dict, but it is of type %s' % (
-                        info_field, type(got).__name__))
-                expected_num = int(expected.partition(':')[2])
-                assertGreaterEqual(
-                    self, len(got), expected_num,
-                    'Expected %d items in field %s, but only got %d' % (
-                        expected_num, info_field, len(got)
-                    )
-                )
-                continue
-            else:
-                got = got_dict.get(info_field)
-            self.assertEqual(expected, got,
-                             'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
+        got = got_dict.get(info_field)
+        expect_value(self, got, expected, info_field)
 
 
 def expect_info_dict(self, got_dict, expected_dict):

From 687c04cbb8d80f76b45caf9377b2d3b64462c203 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Wed, 30 Sep 2015 10:30:04 +0200
Subject: [PATCH 1442/2145] [test] use descriptive variable name

---
 test/helper.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e50d7ff28..5c44105d4 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,15 +128,13 @@ def expect_value(self, got, expected, field):
         self.assertEqual(len(expected), len(got),
                          'Expect a list of length %d, but got a list of length %d' % (
                          len(expected), len(got)))
-        _id = 0
-        for i, j in zip(got, expected):
-            _type_i = type(i)
-            _type_j = type(j)
-            self.assertEqual(_type_j, _type_i,
+        for index, (item_got, item_expected) in enumerate(zip(got, expected)):
+            type_got = type(item_got)
+            type_expected = type(item_expected)
+            self.assertEqual(type_expected, type_got,
                              'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                             _id, field, _type_j, _type_i))
-            expect_value(self, i, j, field)
-            _id += 1
+                             index, field, type_expected, type_got))
+            expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
             got = 'md5:' + md5(got)

From 2e885de796e211300fd70709da17325b84c86dd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:21:01 +0600
Subject: [PATCH 1443/2145] [test/helper] Formatting

---
 test/helper.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 5c44105d4..288ed237d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -120,20 +120,22 @@ def expect_value(self, got, expected, field):
             contains_str in got,
             'field %s (value: %r) should contain %r' % (field, got, contains_str))
     elif isinstance(expected, type):
-        self.assertTrue(isinstance(got, expected),
-                        'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+        self.assertTrue(
+            isinstance(got, expected),
+            'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
     elif isinstance(expected, dict) and isinstance(got, dict):
         expect_dict(self, got, expected)
     elif isinstance(expected, list) and isinstance(got, list):
-        self.assertEqual(len(expected), len(got),
-                         'Expect a list of length %d, but got a list of length %d' % (
-                         len(expected), len(got)))
+        self.assertEqual(
+            len(expected), len(got),
+            'Expect a list of length %d, but got a list of length %d' % (len(expected), len(got)))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)
             type_expected = type(item_expected)
-            self.assertEqual(type_expected, type_got,
-                             'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                             index, field, type_expected, type_got))
+            self.assertEqual(
+                type_expected, type_got,
+                'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
@@ -146,13 +148,11 @@ def expect_value(self, got, expected, field):
             expected_num = int(expected.partition(':')[2])
             assertGreaterEqual(
                 self, len(got), expected_num,
-                'Expected %d items in field %s, but only got %d' % (
-                    expected_num, field, len(got)
-                )
-            )
+                'Expected %d items in field %s, but only got %d' % (expected_num, field, len(got)))
             return
-        self.assertEqual(expected, got,
-                         'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+        self.assertEqual(
+            expected, got,
+            'invalid value for field %s, expected %r, got %r' % (field, expected, got))
 
 
 def expect_dict(self, got_dict, expected_dict):

From 386a7b52d548da0aaf11e5805d67d5ab30d9dfab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:26:42 +0600
Subject: [PATCH 1444/2145] [test/helper] Spelling

---
 test/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 288ed237d..0ce9a0fb3 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -134,7 +134,7 @@ def expect_value(self, got, expected, field):
             type_expected = type(item_expected)
             self.assertEqual(
                 type_expected, type_got,
-                'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
                 index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
@@ -152,7 +152,7 @@ def expect_value(self, got, expected, field):
             return
         self.assertEqual(
             expected, got,
-            'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+            'Invalid value for field %s, expected %r, got %r' % (field, expected, got))
 
 
 def expect_dict(self, got_dict, expected_dict):

From f88f1b40ce7f70a3a510259154364a25d7dceff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:31:29 +0600
Subject: [PATCH 1445/2145] [test/helper] Clarify field for list length
 mismatch

---
 test/helper.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 0ce9a0fb3..28fd135b2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,7 +128,8 @@ def expect_value(self, got, expected, field):
     elif isinstance(expected, list) and isinstance(got, list):
         self.assertEqual(
             len(expected), len(got),
-            'Expect a list of length %d, but got a list of length %d' % (len(expected), len(got)))
+            'Expect a list of length %d, but got a list of length %d for field %s' % (
+                len(expected), len(got), field))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)
             type_expected = type(item_expected)

From 8466336104164b37b3f32c32278e2426590db42f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 22:12:52 +0600
Subject: [PATCH 1446/2145] [vk] Detect vimeo embeds (Closes #7021)

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c30c5a8e5..765e9e6fd 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -17,6 +17,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
 )
+from .vimeo import VimeoIE
 
 
 class VKIE(InfoExtractor):
@@ -249,6 +250,10 @@ class VKIE(InfoExtractor):
         if youtube_url:
             return self.url_result(youtube_url, 'Youtube')
 
+        vimeo_url = VimeoIE._extract_vimeo_url(url, info_page)
+        if vimeo_url is not None:
+            return self.url_result(vimeo_url)
+
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:

From f540b937062a4de2cd0a097245ec82dd9fd23277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 02:33:48 +0600
Subject: [PATCH 1447/2145] [naver] Improve error regex

---
 youtube_dl/extractor/naver.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 925967753..35cbb3e6d 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -46,11 +46,11 @@ class NaverIE(InfoExtractor):
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
-            m_error = re.search(
-                r'(?s)<div class="(?:nation_error|nation_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
-                webpage)
-            if m_error:
-                raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
+            error = self._html_search_regex(
+                r'(?s)<div class="(?:nation_error|nation_box|error_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                webpage, 'error', default=None)
+            if error:
+                raise ExtractorError(error, expected=True)
             raise ExtractorError('couldn\'t extract vid and key')
         vid = m_id.group(1)
         key = m_id.group(2)

From b83831df1fb0dd241210a2c992ac9753c5df59cd Mon Sep 17 00:00:00 2001
From: Joakim Fremstad <jfremstad@users.noreply.github.com>
Date: Thu, 1 Oct 2015 14:58:49 +0200
Subject: [PATCH 1448/2145] [nrk] Spelling

---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d066a96db..5ed235f1e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -49,7 +49,7 @@ class NRKIE(InfoExtractor):
 
         if data['usageRights']['isGeoBlocked']:
             raise ExtractorError(
-                'NRK har ikke rettig-heter til å vise dette programmet utenfor Norge',
+                'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
                 expected=True)
 
         video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'

From 2b3f951a2e2e01adfb068eac05b976ff445db073 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 20:33:17 +0600
Subject: [PATCH 1449/2145] [nrktv] Rework subtitles and eliminate downloading
 twice

---
 youtube_dl/extractor/nrk.py | 28 +++++++++-------------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 5ed235f1e..8ac38a174 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -196,20 +197,6 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
-    def _debug_print(self, txt):
-        if self._downloader.params.get('verbose', False):
-            self.to_screen('[debug] %s' % txt)
-
-    def _get_subtitles(self, subtitlesurl, video_id, baseurl):
-        url = "%s%s" % (baseurl, subtitlesurl)
-        self._debug_print('%s: Subtitle url: %s' % (video_id, url))
-        captions = self._download_xml(
-            url, video_id, 'Downloading subtitles')
-        lang = captions.get('lang', 'no')
-        return {lang: [
-            {'ext': 'ttml', 'url': url},
-        ]}
-
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(
             manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
@@ -218,7 +205,7 @@ class NRKTVIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         part_id = mobj.group('part_id')
-        baseurl = mobj.group('baseurl')
+        base_url = mobj.group('baseurl')
 
         webpage = self._download_webpage(url, video_id)
 
@@ -278,11 +265,14 @@ class NRKTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         subtitles_url = self._html_search_regex(
-            r'data-subtitlesurl[ ]*=[ ]*"([^"]+)"',
-            webpage, 'subtitle URL', default=None)
-        subtitles = None
+            r'data-subtitlesurl\s*=\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'subtitle URL', default=None, group='url')
+        subtitles = {}
         if subtitles_url:
-            subtitles = self.extract_subtitles(subtitles_url, video_id, baseurl)
+            subtitles['no'] = [{
+                'ext': 'ttml',
+                'url': compat_urlparse.urljoin(base_url, subtitles_url),
+            }]
 
         return {
             'id': video_id,

From e7d8e98a9ffdec2502bedb21a4f043df6da225a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 21:51:04 +0600
Subject: [PATCH 1450/2145] [extractor/common] Allow float bitrates

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2fe0d5d37..9c40d56a9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1084,7 +1084,7 @@ class InfoExtractor(object):
             if not src:
                 continue
 
-            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
             filesize = int_or_none(video.get('size') or video.get('fileSize'))
             width = int_or_none(video.get('width'))
             height = int_or_none(video.get('height'))

From 1e5bcdec0264190ed2a05ee49c1f9f5b20ba3aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:08:16 +0600
Subject: [PATCH 1451/2145] [extractor/common] Extract images from SMIL

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c40d56a9..5684227dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1054,10 +1054,18 @@ class InfoExtractor(object):
             elif not description and name in ('description', 'abstract'):
                 description = content
 
+        thumbnails = [{
+            'id': image.get('type'),
+            'url': image.get('src'),
+            'width': int_or_none(image.get('width')),
+            'height': int_or_none(image.get('height')),
+        } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
+
         return {
             'id': video_id,
             'title': title or video_id,
             'description': description,
+            'thumbnails': thumbnails,
             'formats': formats,
             'subtitles': subtitles,
         }

From 647eab4541d1f55dea6aa20af9b35c2726dda48a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:18:59 +0600
Subject: [PATCH 1452/2145] [extractor/common] Extract upload date from SMIL

---
 youtube_dl/extractor/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5684227dc..b86d06523 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -39,6 +39,7 @@ from ..utils import (
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
+    unified_strdate,
     url_basename,
     xpath_text,
     xpath_with_ns,
@@ -1044,6 +1045,7 @@ class InfoExtractor(object):
         video_id = os.path.splitext(url_basename(smil_url))[0]
         title = None
         description = None
+        upload_date = None
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
             name = meta.attrib.get('name')
             content = meta.attrib.get('content')
@@ -1053,6 +1055,8 @@ class InfoExtractor(object):
                 title = content
             elif not description and name in ('description', 'abstract'):
                 description = content
+            elif not upload_date and name == 'date':
+                upload_date = unified_strdate(content)
 
         thumbnails = [{
             'id': image.get('type'),
@@ -1065,6 +1069,7 @@ class InfoExtractor(object):
             'id': video_id,
             'title': title or video_id,
             'description': description,
+            'upload_date': upload_date,
             'thumbnails': thumbnails,
             'formats': formats,
             'subtitles': subtitles,

From acfb717a18cfa9c0f377372068e16862bba345b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:19:39 +0600
Subject: [PATCH 1453/2145] [videolecturesnet] Use generic SMIL extraction

---
 youtube_dl/extractor/videolecturesnet.py | 62 +++---------------------
 1 file changed, 8 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index ef2da5632..160dbb590 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -1,13 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    find_xpath_attr,
-    int_or_none,
     parse_duration,
-    unified_strdate,
 )
 
 
@@ -29,58 +24,17 @@ class VideoLecturesNetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_xml(smil_url, video_id)
+        smil = self._download_smil(smil_url, video_id)
 
-        title = find_xpath_attr(smil, './/meta', 'name', 'title').attrib['content']
-        description_el = find_xpath_attr(smil, './/meta', 'name', 'abstract')
-        description = (
-            None if description_el is None
-            else description_el.attrib['content'])
-        upload_date = unified_strdate(
-            find_xpath_attr(smil, './/meta', 'name', 'date').attrib['content'])
+        info = self._parse_smil(smil, smil_url, video_id)
+
+        info['id'] = video_id
 
         switch = smil.find('.//switch')
-        duration = parse_duration(switch.attrib.get('dur'))
-        thumbnail_el = find_xpath_attr(switch, './image', 'type', 'thumbnail')
-        thumbnail = (
-            None if thumbnail_el is None else thumbnail_el.attrib.get('src'))
+        if switch is not None:
+            info['duration'] = parse_duration(switch.attrib.get('dur'))
 
-        formats = []
-        for v in switch.findall('./video'):
-            proto = v.attrib.get('proto')
-            if proto not in ['http', 'rtmp']:
-                continue
-            f = {
-                'width': int_or_none(v.attrib.get('width')),
-                'height': int_or_none(v.attrib.get('height')),
-                'filesize': int_or_none(v.attrib.get('size')),
-                'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
-                'ext': v.attrib.get('ext'),
-            }
-            src = v.attrib['src']
-            if proto == 'http':
-                if self._is_valid_url(src, video_id):
-                    f['url'] = src
-                    formats.append(f)
-            elif proto == 'rtmp':
-                f.update({
-                    'url': v.attrib['streamer'],
-                    'play_path': src,
-                    'rtmp_real_time': True,
-                })
-                formats.append(f)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'upload_date': upload_date,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
+        return info

From 0c996b9f488bfaa74d79e94739af80a0be38e125 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:39:38 +0600
Subject: [PATCH 1454/2145] [videolecturesnet] Add support for playlists
 (Closes #7031)

---
 youtube_dl/extractor/videolecturesnet.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 160dbb590..eadff8d18 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-)
+from ..compat import compat_urlparse
+from ..utils import parse_duration
 
 
 class VideoLecturesNetIE(InfoExtractor):
@@ -27,7 +28,17 @@ class VideoLecturesNetIE(InfoExtractor):
         video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_smil(smil_url, video_id)
+        smil = self._download_smil(smil_url, video_id, fatal=False)
+
+        # Probably a playlist
+        if smil is False:
+            webpage = self._download_webpage(url, video_id)
+            entries = [
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+            playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
+            playlist_description = self._html_search_meta('description', webpage, 'description')
+            return self.playlist_result(entries, video_id, playlist_title, playlist_description)
 
         info = self._parse_smil(smil, smil_url, video_id)
 

From fb97809e64a145768346345ff2b5e5539880c014 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:44:51 +0600
Subject: [PATCH 1455/2145] [videolecturesnet] Improve playlist extraction

---
 youtube_dl/extractor/videolecturesnet.py | 32 +++++++++++++++---------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index eadff8d18..704165406 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -3,8 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import parse_duration
+from ..compat import (
+    compat_HTTPError,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    parse_duration,
+)
 
 
 class VideoLecturesNetIE(InfoExtractor):
@@ -28,17 +34,19 @@ class VideoLecturesNetIE(InfoExtractor):
         video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_smil(smil_url, video_id, fatal=False)
 
-        # Probably a playlist
-        if smil is False:
-            webpage = self._download_webpage(url, video_id)
-            entries = [
-                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
-            playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
-            playlist_description = self._html_search_meta('description', webpage, 'description')
-            return self.playlist_result(entries, video_id, playlist_title, playlist_description)
+        try:
+            smil = self._download_smil(smil_url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                # Probably a playlist
+                webpage = self._download_webpage(url, video_id)
+                entries = [
+                    self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                    for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+                playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
+                playlist_description = self._html_search_meta('description', webpage, 'description')
+                return self.playlist_result(entries, video_id, playlist_title, playlist_description)
 
         info = self._parse_smil(smil, smil_url, video_id)
 

From 6edaa0e25b880be5d1ea513d9d907c91e18a8347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:45:10 +0600
Subject: [PATCH 1456/2145] [videolecturesnet] Add playlist test

---
 youtube_dl/extractor/videolecturesnet.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 704165406..113a2289b 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -17,7 +17,7 @@ class VideoLecturesNetIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
             'id': 'promogram_igor_mekjavic_eng',
@@ -28,7 +28,15 @@ class VideoLecturesNetIE(InfoExtractor):
             'duration': 565,
             'thumbnail': 're:http://.*\.jpg',
         },
-    }
+    }, {
+        'url': 'http://videolectures.net/deeplearning2015_montreal/',
+        'info_dict': {
+            'id': 'deeplearning2015_montreal',
+            'title': 'Deep Learning Summer School, Montreal 2015',
+            'description': 'md5:90121a40cc6926df1bf04dcd8563ed3b',
+        },
+        'playlist_count': 30,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c78e48177c72baa0c985c4d47f95e6edc6ec33dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:54:54 +0600
Subject: [PATCH 1457/2145] [extractor/common] Check validity of direct URLs

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b86d06523..b928e24be 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1145,7 +1145,7 @@ class InfoExtractor(object):
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
                 continue
 
-            if src_url.startswith('http'):
+            if src_url.startswith('http') and self._is_valid_url(src, video_id):
                 http_count += 1
                 formats.append({
                     'url': src_url,

From 3a1341a7bc9f11d972c7649a33e143fab72f8bc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:59:20 +0600
Subject: [PATCH 1458/2145] [extractor/common] Make m3u8 extraction for SMIL
 non fatal

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b928e24be..9a5a7cc2c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1129,8 +1129,10 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls'))
+                m3u8_formats = self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
                 continue
 
             if src_ext == 'f4m':

From 4de6131090e0232c7cc99bcaafe6a3e71269b7af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:03:31 +0600
Subject: [PATCH 1459/2145] [extractor/common] Add fatal to
 _extract_f4m_formats

---
 youtube_dl/extractor/common.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9a5a7cc2c..c2aa3a749 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -870,13 +870,18 @@ class InfoExtractor(object):
         time.sleep(timeout)
 
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
-                             transform_source=lambda s: fix_xml_ampersands(s).strip()):
+                             transform_source=lambda s: fix_xml_ampersands(s).strip(),
+                             fatal=True):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
-            transform_source=transform_source)
+            transform_source=transform_source,
+            fatal=fatal)
+
+        if manifest is False:
+            return manifest
 
         formats = []
         manifest_version = '1.0'
@@ -897,7 +902,10 @@ class InfoExtractor(object):
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
-                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    f4m_formats = self._extract_f4m_formats(
+                        manifest_url, video_id, preference, f4m_id, fatal=fatal)
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

From e5851b963ab0872f27f2db1cff3c89548b6bf95e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:04:56 +0600
Subject: [PATCH 1460/2145] [extractor/common] Make f4m extraction for SMIL non
 fatal

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c2aa3a749..4fe2307cd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1152,7 +1152,9 @@ class InfoExtractor(object):
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)
-                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
+                f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):

From 06c6efa9701d0527147875adf0caf437ffbb0397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:10:36 +0600
Subject: [PATCH 1461/2145] [videolecturesnet] Add test video with broken
 direct format links

---
 youtube_dl/extractor/videolecturesnet.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 113a2289b..649ac9433 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -28,6 +28,21 @@ class VideoLecturesNetIE(InfoExtractor):
             'duration': 565,
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        # video with invalid direct format links (HTTP 403)
+        'url': 'http://videolectures.net/russir2010_filippova_nlp/',
+        'info_dict': {
+            'id': 'russir2010_filippova_nlp',
+            'ext': 'flv',
+            'title': 'NLP at Google',
+            'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
+            'duration': 5352,
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {

From 44451f22d5d1a5bed5f5851b27a963860813ecd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 13:41:52 +0200
Subject: [PATCH 1462/2145] [naver] Remove unused import

---
 youtube_dl/extractor/naver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 35cbb3e6d..1f5fc2145 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -10,7 +10,6 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
-    clean_html,
 )
 
 

From 7d0ada5ff907824c66c466ee9b83008210250d5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 13:42:11 +0200
Subject: [PATCH 1463/2145] [test/helper] Fix style

Use the correct indentation to please flake8
---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 28fd135b2..bdd7acca4 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -136,7 +136,7 @@ def expect_value(self, got, expected, field):
             self.assertEqual(
                 type_expected, type_got,
                 'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
-                index, field, type_expected, type_got))
+                    index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):

From 0facd2af3ebfda68b79c7e2e1c575d73f9680802 Mon Sep 17 00:00:00 2001
From: fluks <fluks.github@gmail.com>
Date: Fri, 2 Oct 2015 04:08:13 +0300
Subject: [PATCH 1464/2145] Fix ruutu extractor bug

If there's no resolution attribute in xml, only width gets a
value, height doesn't and ValueError is raised.
---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index c67ad25ce..7720f1383 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -74,7 +74,7 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
-                        width, height = [int_or_none(x) for x in child.get('resolution', '').split('x')]
+                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')]
                         formats.append({
                             'format_id': '%s-%s' % (proto, label if label else tbr),
                             'url': video_url,

From 59a9efe85b15e53c5928b7fdb810c150f5bf4b78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 20:48:39 +0600
Subject: [PATCH 1465/2145] [ruutu] Limit resolution split to 2 pieces (Closes
 #7037, closes #7042)

---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 7720f1383..a16b73ff4 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -74,7 +74,7 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
-                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')]
+                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]]
                         formats.append({
                             'format_id': '%s-%s' % (proto, label if label else tbr),
                             'url': video_url,

From 3bb3f0410822d3d21c6199bb8915b598990628e6 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 16:59:04 +0200
Subject: [PATCH 1466/2145] [europa] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/europa.py   | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/europa.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..495a18c17 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -158,6 +158,7 @@ from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .espn import ESPNIE
 from .esri import EsriVideoIE
+from .europa import EuropaIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
new file mode 100644
index 000000000..c437c4886
--- /dev/null
+++ b/youtube_dl/extractor/europa.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urlparse,
+    xpath_text
+)
+
+
+class EuropaIE(InfoExtractor):
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?(?:[^&]|&(?!ref))*ref=(?P<id>[A-Za-z0-9]+)'
+    _TEST = {
+        'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
+        'md5': '728cca2fd41d5aa7350cec1141fbe620',
+        'info_dict': {
+            'id': 'I107758',
+            'ext': 'mp4',
+            'title': 'TRADE - Wikileaks on TTIP',
+            'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
+            'thumbnail': 're:^http://defiris\.ec\.streamcloud\.be/findmedia/18/107758/THUMB_[0-9A-Z]+\.jpg$'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        lang = query.get('sitelang', ['en'])[0]
+
+        playlist = self._download_xml('http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=' + video_id, video_id)
+        videos = {}
+        formats = []
+
+        for item in playlist.findall('info/title/item'):
+            videos[xpath_text(item, 'lg')] = {'title': xpath_text(item, 'label').strip()}
+
+        for item in playlist.findall('info/description/item'):
+            videos[xpath_text(item, 'lg')]['description'] = xpath_text(item, 'label').strip()
+
+        for item in playlist.findall('files/file'):
+            lg = xpath_text(item, 'lg')
+            vid = videos[lg]
+            vid['format_note'] = xpath_text(item, 'lglabel')
+            vid['url'] = xpath_text(item, 'url')
+
+            if lg == lang:
+                vid['language_preference'] = 10
+
+            formats.append(vid)
+
+        formats.reverse()
+        def_video = videos.get(lang, videos['int'])
+
+        return {
+            'id': video_id,
+            'title': def_video['title'],
+            'description': def_video['description'],
+            'thumbnail': xpath_text(playlist, 'info/thumburl', 'thumburl'),
+            'formats': formats
+        }

From 8fc226ef994a82f7b1050cdb72ec38922d3ab9cf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 2 Oct 2015 17:24:30 +0100
Subject: [PATCH 1467/2145] [nba] extract all video formats and extract more
 info

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/nba.py      | 102 +++++++++++++++++++++----------
 2 files changed, 74 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..78478b38b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -351,7 +351,10 @@ from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
-from .nba import NBAIE
+from .nba import (
+    NBAIE,
+    NBAWatchIE,
+)
 from .nbc import (
     NBCIE,
     NBCNewsIE,
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 944096e1c..36ece5b64 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -2,62 +2,100 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    remove_end,
     parse_duration,
+    parse_iso8601,
+    int_or_none,
 )
 
 
-class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+class NBABaseIE(InfoExtractor):
+    def _get_formats(self, video_id):
+        base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id
+        return [{
+            'url': base_url + '_nba_android_high.mp4',
+            'width': 480,
+            'height': 320,
+            'format_id': '320p',
+        },{
+            'url': base_url + '_640x360_664b.mp4',
+            'width': 640,
+            'height': 360,
+            'format_id': '360p',
+        },{
+            'url': base_url + '_768x432_1404.mp4',
+            'width': 768,
+            'height': 432,
+            'format_id': '432p',
+        },{
+            'url': base_url + '_1280x720.mp4',
+            'width': 1280,
+            'height': 720,
+            'format_id': '720p',
+        }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        ret = self._extract_metadata(webpage, video_id)
+        ret['id'] = video_id.rpartition('/')[2]
+        ret['formats'] = self._get_formats(video_id)
+        return ret
+
+
+class NBAIE(NBABaseIE):
+    IE_NAME = 'nba'
+    _VALID_URL = r'https?://(?:www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
-        'md5': 'c0edcfc37607344e2ff8f13c378c88a4',
+        'md5': '9d902940d2a127af3f7f9d2f3dc79c96',
         'info_dict': {
             'id': '0021200253-okc-bkn-recap.nba',
             'ext': 'mp4',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
+            'timestamp': 1354680189,
+            'upload_date': '20121205',
         },
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
-    }, {
+    }]
+
+    def _extract_metadata(self, webpage, video_id):
+        return {
+            'title': self._html_search_meta('name', webpage),
+            'description': self._html_search_meta('description', webpage),
+            'duration': parse_duration(self._html_search_meta('duration', webpage)),
+            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
+            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage))
+        }
+
+class NBAWatchIE(NBABaseIE):
+    IE_NAME = 'nba:watch'
+    _VALID_URL = r'https?://watch.nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+    _TESTS = [{
         'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
             'id': '0041400301-cle-atl-recap.nba',
             'ext': 'mp4',
-            'title': 'NBA GAME TIME | Video: Hawks vs. Cavaliers Game 1',
+            'title': 'Hawks vs. Cavaliers Game 1',
             'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
             'duration': 228,
-        },
-        'params': {
-            'skip_download': True,
+            'timestamp': 1432094400,
+            'upload_date': '20150520',
         }
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
-
-        shortened_video_id = video_id.rpartition('/')[2]
-        title = remove_end(
-            self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
-
-        description = self._og_search_description(webpage)
-        duration_str = self._html_search_meta(
-            'duration', webpage, 'duration', default=None)
-        if not duration_str:
-            duration_str = self._html_search_regex(
-                r'Duration:</b>\s*(\d+:\d+)', webpage, 'duration', fatal=False)
-        duration = parse_duration(duration_str)
-
+    def _extract_metadata(self, webpage, video_id):
+        program_id = self._search_regex(r'var\s+programId\s*=\s*(\d+);', webpage, 'program id')
+        metadata = self._download_json(
+            'http://smbsolr.cdnak.neulion.com/solr_nbav6/nba/nba/mlt/?wt=json&fl=name,description,image,runtime,releaseDate&q=sequence%3A' + program_id, video_id)['match']['docs'][0]
         return {
-            'id': shortened_video_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'duration': duration,
+            'title': metadata['name'],
+            'description': metadata.get('description'),
+            'duration': int_or_none(metadata.get('runtime')),
+            'thumbnail': metadata.get('image'),
+            'timestamp': parse_iso8601(metadata.get('releaseDate'))
         }

From af17794c654bd24bbd5f47997596430b201ea08e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 22:29:15 +0600
Subject: [PATCH 1468/2145] [europa] Improve extraction

---
 youtube_dl/extractor/europa.py | 94 ++++++++++++++++++++++------------
 1 file changed, 62 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index c437c4886..02ba8d63c 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -2,59 +2,89 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
+    int_or_none,
+    orderedSet,
+    parse_duration,
+    qualities,
+    unified_strdate,
     xpath_text
 )
 
 
 class EuropaIE(InfoExtractor):
-    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?(?:[^&]|&(?!ref))*ref=(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9]+)'
+    _TESTS = [{
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
-        'md5': '728cca2fd41d5aa7350cec1141fbe620',
+        'md5': '574f080699ddd1e19a675b0ddf010371',
         'info_dict': {
             'id': 'I107758',
             'ext': 'mp4',
             'title': 'TRADE - Wikileaks on TTIP',
             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
-            'thumbnail': 're:^http://defiris\.ec\.streamcloud\.be/findmedia/18/107758/THUMB_[0-9A-Z]+\.jpg$'
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150811',
+            'duration': 34,
+            'view_count': int,
+            'formats': 'mincount:3',
         }
-    }
+    }, {
+        'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        playlist = self._download_xml(
+            'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=%s' % video_id, video_id)
+
+        def get_item(type_, preference):
+            items = {}
+            for item in playlist.findall('./info/%s/item' % type_):
+                lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
+                if lang and label:
+                    items[lang] = label.strip()
+            for p in preference:
+                if items.get(p):
+                    return items[p]
+
         query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        lang = query.get('sitelang', ['en'])[0]
+        preferred_lang = query.get('sitelang', ('en', ))[0]
+
+        preferred_langs = orderedSet((preferred_lang, 'en', 'int'))
+
+        title = get_item('title', preferred_langs) or video_id
+        description = get_item('description', preferred_langs)
+        thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
+        upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
+        duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
+        view_count = int_or_none(xpath_text(playlist,'./info/views', 'views'))
+
+        language_preference = qualities(preferred_langs[::-1])
 
-        playlist = self._download_xml('http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=' + video_id, video_id)
-        videos = {}
         formats = []
-
-        for item in playlist.findall('info/title/item'):
-            videos[xpath_text(item, 'lg')] = {'title': xpath_text(item, 'label').strip()}
-
-        for item in playlist.findall('info/description/item'):
-            videos[xpath_text(item, 'lg')]['description'] = xpath_text(item, 'label').strip()
-
-        for item in playlist.findall('files/file'):
-            lg = xpath_text(item, 'lg')
-            vid = videos[lg]
-            vid['format_note'] = xpath_text(item, 'lglabel')
-            vid['url'] = xpath_text(item, 'url')
-
-            if lg == lang:
-                vid['language_preference'] = 10
-
-            formats.append(vid)
-
-        formats.reverse()
-        def_video = videos.get(lang, videos['int'])
+        for file_ in playlist.findall('./files/file'):
+            video_url = xpath_text(file_, './url')
+            if not video_url:
+                continue
+            lang = xpath_text(file_, './lg')
+            formats.append({
+                'url': video_url,
+                'format_id': lang,
+                'format_note': xpath_text(file_, './lglabel'),
+                'language_preference': language_preference(lang)
+            })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': def_video['title'],
-            'description': def_video['description'],
-            'thumbnail': xpath_text(playlist, 'info/thumburl', 'thumburl'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnmail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
             'formats': formats
         }

From f3b098fb90b985484d800fcdbfe18add2360e4df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 23:22:53 +0600
Subject: [PATCH 1469/2145] [europa] Add support for audio URLs

---
 youtube_dl/extractor/europa.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index 02ba8d63c..9e33cacff 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class EuropaIE(InfoExtractor):
-    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
         'md5': '574f080699ddd1e19a675b0ddf010371',
@@ -32,6 +32,9 @@ class EuropaIE(InfoExtractor):
     }, {
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
         'only_matching': True,
+    }, {
+        'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From b203095d4c95d471bc2ac7045693c6938ee914d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 22:40:35 +0200
Subject: [PATCH 1470/2145] [europa] Style fix: add whitespace after comma

---
 youtube_dl/extractor/europa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index 9e33cacff..adc43919e 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -63,7 +63,7 @@ class EuropaIE(InfoExtractor):
         thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
         upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
         duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
-        view_count = int_or_none(xpath_text(playlist,'./info/views', 'views'))
+        view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))
 
         language_preference = qualities(preferred_langs[::-1])
 

From 5495937f461268a850a6a54d3fe19ed1f0f01eef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:00:10 +0800
Subject: [PATCH 1471/2145] [options] Cleanup double spaces in help texts

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5eccc0a70..3dd6d290b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -276,7 +276,7 @@ def parseOpts(overrideArguments=None):
             'For example, to only match videos that have been liked more than '
             '100 times and disliked less than 50 times (or the dislike '
             'functionality is not available at the given service), but who '
-            'also have a description, use  --match-filter '
+            'also have a description, use --match-filter '
             '"like_count > 100 & dislike_count <? 50 & description" .'
         ))
     selection.add_option(
@@ -602,7 +602,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
+        help='[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From 13118a50b88c26fe5d932eccf77e297249a78634 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:04:27 +0800
Subject: [PATCH 1472/2145] [compat] Allow overriding by only COLUMNS or LINES
 in compat_get_terminal_size

Now the semantic of this function is identical to
shutil.get_terminal_size() in Python 3.3+. The new behavior also
corresponds to the old get_term_width(), which is removed in
003c69a84b68cadb46aeb8e03115848a722fd675
---
 youtube_dl/compat.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1ff42d94b..c36c9c23f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -416,7 +416,7 @@ if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
 else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
-    def compat_get_terminal_size():
+    def compat_get_terminal_size(fallback=(80, 24)):
         columns = compat_getenv('COLUMNS', None)
         if columns:
             columns = int(columns)
@@ -428,14 +428,20 @@ else:
         else:
             lines = None
 
-        try:
-            sp = subprocess.Popen(
-                ['stty', 'size'],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            out, err = sp.communicate()
-            lines, columns = map(int, out.split())
-        except Exception:
-            pass
+        if columns <= 0 or lines <= 0:
+            try:
+                sp = subprocess.Popen(
+                    ['stty', 'size'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                out, err = sp.communicate()
+                _columns, _lines = map(int, out.split())
+            except Exception:
+                _columns, _lines = _terminal_size(*fallback)
+
+            if columns <= 0:
+                columns = _columns
+            if lines <= 0:
+                lines = _lines
         return _terminal_size(columns, lines)
 
 try:

From bad84757eb135b85d5a1b29524a064d23ab4e1e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:10:23 +0800
Subject: [PATCH 1473/2145] [doc] Better formatting of youtube-dl.1 (closes
 #6510)

---
 devscripts/prepare_manpage.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 7ece37754..776e6556e 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -8,6 +8,35 @@ import re
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
+
+def filter_options(readme):
+    ret = ''
+    in_options = False
+    for line in readme.split('\n'):
+        if line.startswith('# '):
+            if line[2:].startswith('OPTIONS'):
+                in_options = True
+            else:
+                in_options = False
+
+        if in_options:
+            if line.lstrip().startswith('-'):
+                option, description = re.split(r'\s{2,}', line.lstrip())
+                split_option = option.split(' ')
+
+                if not split_option[-1].startswith('-'):  # metavar
+                    option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])
+
+                # Pandoc's definition_lists. See http://pandoc.org/README.html
+                # for more information.
+                ret += '\n%s\n:   %s\n' % (option, description)
+            else:
+                ret += line.lstrip() + '\n'
+        else:
+            ret += line + '\n'
+
+    return ret
+
 with io.open(README_FILE, encoding='utf-8') as f:
     readme = f.read()
 
@@ -26,6 +55,8 @@ readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
 readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
 readme = PREFIX + readme
 
+readme = filter_options(readme)
+
 if sys.version_info < (3, 0):
     print(readme.encode('utf-8'))
 else:

From 28809ab07a8d10f9cafc3d712414c7b355c27166 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 09:47:19 +0100
Subject: [PATCH 1474/2145] [nba] extract more formats

---
 youtube_dl/extractor/nba.py | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 36ece5b64..8844b61a5 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -10,28 +10,60 @@ from ..utils import (
 
 class NBABaseIE(InfoExtractor):
     def _get_formats(self, video_id):
+        formats = self._extract_m3u8_formats(
+            'http://nbavod-f.akamaihd.net/i/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/master.m3u8' % video_id,
+            video_id,
+            m3u8_id='hls')
+        formats.extend(self._extract_f4m_formats(
+            'http://nbavod-f.akamaihd.net/z/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/manifest.f4m?hdcore=3.4.1.1' % video_id,
+            video_id,
+            f4m_id='hds'))
         base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id
-        return [{
+        formats.extend([{
+            'url': base_url + '_nba_ipad.mp4',
+            'width': 400,
+            'height': 224,
+            'format_id': '224p',
+            'preference': 1,
+        },{
             'url': base_url + '_nba_android_high.mp4',
             'width': 480,
             'height': 320,
             'format_id': '320p',
+            'preference': 2,
+        },{
+            'url': base_url + '_nba_576x324.mp4',
+            'width': 576,
+            'height': 324,
+            'format_id': '324p',
+            'preference': 3,
         },{
             'url': base_url + '_640x360_664b.mp4',
             'width': 640,
             'height': 360,
             'format_id': '360p',
+            'preference': 4,
         },{
             'url': base_url + '_768x432_1404.mp4',
             'width': 768,
             'height': 432,
             'format_id': '432p',
+            'preference': 5,
+        },{
+            'url': base_url + '_960x540_2104.mp4',
+            'width': 960,
+            'height': 540,
+            'format_id': '540p',
+            'preference': 6,
         },{
             'url': base_url + '_1280x720.mp4',
             'width': 1280,
             'height': 720,
             'format_id': '720p',
-        }]
+            'preference': 7,
+        }])
+        self._sort_formats(formats)
+        return formats
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c233e6bcc398f9734d7138854978c1cb00fe757f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 12:30:05 +0100
Subject: [PATCH 1475/2145] [nba] extract video info from xml feed

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/nba.py      | 224 +++++++++++++++++--------------
 2 files changed, 126 insertions(+), 103 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 78478b38b..a73a1317e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -351,10 +351,7 @@ from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
-from .nba import (
-    NBAIE,
-    NBAWatchIE,
-)
+from .nba import NBAIE
 from .nbc import (
     NBCIE,
     NBCNewsIE,
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 8844b61a5..3d38d080e 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -3,131 +3,157 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
-    parse_iso8601,
     int_or_none,
 )
 
 
-class NBABaseIE(InfoExtractor):
-    def _get_formats(self, video_id):
-        formats = self._extract_m3u8_formats(
-            'http://nbavod-f.akamaihd.net/i/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/master.m3u8' % video_id,
-            video_id,
-            m3u8_id='hls')
-        formats.extend(self._extract_f4m_formats(
-            'http://nbavod-f.akamaihd.net/z/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/manifest.f4m?hdcore=3.4.1.1' % video_id,
-            video_id,
-            f4m_id='hds'))
-        base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id
-        formats.extend([{
-            'url': base_url + '_nba_ipad.mp4',
-            'width': 400,
-            'height': 224,
-            'format_id': '224p',
-            'preference': 1,
-        },{
-            'url': base_url + '_nba_android_high.mp4',
-            'width': 480,
-            'height': 320,
-            'format_id': '320p',
-            'preference': 2,
-        },{
-            'url': base_url + '_nba_576x324.mp4',
-            'width': 576,
-            'height': 324,
-            'format_id': '324p',
-            'preference': 3,
-        },{
-            'url': base_url + '_640x360_664b.mp4',
-            'width': 640,
-            'height': 360,
-            'format_id': '360p',
-            'preference': 4,
-        },{
-            'url': base_url + '_768x432_1404.mp4',
-            'width': 768,
-            'height': 432,
-            'format_id': '432p',
-            'preference': 5,
-        },{
-            'url': base_url + '_960x540_2104.mp4',
-            'width': 960,
-            'height': 540,
-            'format_id': '540p',
-            'preference': 6,
-        },{
-            'url': base_url + '_1280x720.mp4',
-            'width': 1280,
-            'height': 720,
-            'format_id': '720p',
-            'preference': 7,
-        }])
-        self._sort_formats(formats)
-        return formats
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        ret = self._extract_metadata(webpage, video_id)
-        ret['id'] = video_id.rpartition('/')[2]
-        ret['formats'] = self._get_formats(video_id)
-        return ret
-
-
-class NBAIE(NBABaseIE):
-    IE_NAME = 'nba'
-    _VALID_URL = r'https?://(?:www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+class NBAIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video/(?P<id>[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': '9d902940d2a127af3f7f9d2f3dc79c96',
         'info_dict': {
-            'id': '0021200253-okc-bkn-recap.nba',
+            'id': '0021200253-okc-bkn-recap',
             'ext': 'mp4',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
-            'timestamp': 1354680189,
-            'upload_date': '20121205',
+            'timestamp': 1354638466,
+            'upload_date': '20121204',
         },
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
-    }]
-
-    def _extract_metadata(self, webpage, video_id):
-        return {
-            'title': self._html_search_meta('name', webpage),
-            'description': self._html_search_meta('description', webpage),
-            'duration': parse_duration(self._html_search_meta('duration', webpage)),
-            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
-            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage))
-        }
-
-class NBAWatchIE(NBABaseIE):
-    IE_NAME = 'nba:watch'
-    _VALID_URL = r'https?://watch.nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
-    _TESTS = [{
+    },{
         'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
-            'id': '0041400301-cle-atl-recap.nba',
+            'id': '0041400301-cle-atl-recap',
             'ext': 'mp4',
             'title': 'Hawks vs. Cavaliers Game 1',
             'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
             'duration': 228,
-            'timestamp': 1432094400,
+            'timestamp': 1432134543,
             'upload_date': '20150520',
         }
     }]
 
-    def _extract_metadata(self, webpage, video_id):
-        program_id = self._search_regex(r'var\s+programId\s*=\s*(\d+);', webpage, 'program id')
-        metadata = self._download_json(
-            'http://smbsolr.cdnak.neulion.com/solr_nbav6/nba/nba/mlt/?wt=json&fl=name,description,image,runtime,releaseDate&q=sequence%3A' + program_id, video_id)['match']['docs'][0]
+    _BASE_PATHS = {
+        'turner': 'http://nba.cdn.turner.com/nba/big',
+        'akamai': 'http://nbavod-f.akamaihd.net',
+    }
+
+    _QUALITIES = {
+        '420mp4': {
+            'width': 400,
+            'height': 224,
+            'preference': 1,
+        },
+        '416x234': {
+            'width': 416,
+            'height': 234,
+            'preference': 2,
+        },
+        '556': {
+            'width': 416,
+            'height': 234,
+            'preference': 3,
+        },
+        '480x320_910': {
+            'width': 480,
+            'height': 320,
+            'preference': 4,
+        },
+        'nba_576x324': {
+            'width': 576,
+            'height': 324,
+            'preference': 5,
+        },
+        'nba_640x360': {
+            'width': 640,
+            'height': 360,
+            'preference': 6,
+        },
+        '640x360_664b': {
+            'width': 640,
+            'height': 360,
+            'preference': 7,
+        },
+        '640x360_664m': {
+            'width': 640,
+            'height': 360,
+            'preference': 8,
+        },
+        '768x432_996': {
+            'width': 768,
+            'height': 432,
+            'preference': 9,
+        },
+        '768x432_1404': {
+            'width': 768,
+            'height': 432,
+            'preference': 10,
+        },
+        '960x540_2104': {
+            'width': 960,
+            'height': 540,
+            'preference': 11,
+        },
+        '1280x720_3072': {
+            'width': 1280,
+            'height': 720,
+            'preference': 12,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id)
+        video_id = video_info.find('slug').text
+        title = video_info.find('headline').text
+        description = video_info.find('description').text
+        duration = parse_duration(video_info.find('length').text)
+        timestamp = int_or_none(video_info.find('dateCreated').attrib.get('uts'))
+
+        thumbnails = []
+        for image in video_info.find('images'):
+            thumbnails.append({
+                'id': image.attrib.get('cut'),
+                'url': image.text,
+                'width': int_or_none(image.attrib.get('width')),
+                'height': int_or_none(image.attrib.get('height')),
+            })
+
+        formats = []
+        for video_file in video_info.find('files').iter('file'):
+            video_url = video_file.text
+            if not video_url.startswith('http://'):
+                if video_url.endswith('.m3u8') or video_url.endswith('.f4m'):
+                    video_url = self._BASE_PATHS['akamai'] + video_url
+                else:
+                    video_url = self._BASE_PATHS['turner'] + video_url
+            if video_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(video_url, video_id))
+            elif video_url.endswith('.f4m'):
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id))
+            else:
+                key = video_file.attrib.get('bitrate')
+                quality = self._QUALITIES[key]
+                formats.append({
+                    'format_id': key,
+                    'url': video_url,
+                    'width': quality['width'],
+                    'height': quality['height'],
+                    'preference': quality['preference'],
+                })
+        self._sort_formats(formats)
+
         return {
-            'title': metadata['name'],
-            'description': metadata.get('description'),
-            'duration': int_or_none(metadata.get('runtime')),
-            'thumbnail': metadata.get('image'),
-            'timestamp': parse_iso8601(metadata.get('releaseDate'))
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'thumbnails': thumbnails,
+            'formats': formats,
         }

From 97d5bfcba65c8575ab06a34e91fae30a5fda3161 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 14:17:17 +0100
Subject: [PATCH 1476/2145] [engadget] accept short video urls

---
 youtube_dl/extractor/engadget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index 4ea37ebd9..e4180701d 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class EngadgetIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://www.engadget.com/
-        (?:video/5min/(?P<id>\d+)|
+        (?:video(?:/5min)?/(?P<id>\d+)|
             [\d/]+/.*?)
         '''
 

From 60d23e5e592aebe4a77dfb4ab70e87337967721c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 3 Oct 2015 16:25:33 +0300
Subject: [PATCH 1477/2145] [tapely] Improve _VALID_URL

---
 youtube_dl/extractor/tapely.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index f1f43d0a7..744f9db38 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class TapelyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tape\.ly/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?(?:tape\.ly|tapely\.com)/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
     _API_URL = 'http://tape.ly/showtape?id={0:}'
     _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
     _SOUNDCLOUD_SONG_URL = 'http://api.soundcloud.com{0:}'
@@ -42,6 +42,10 @@ class TapelyIE(InfoExtractor):
                 'ext': 'm4a',
             },
         },
+        {
+            'url': 'https://tapely.com/my-grief-as-told-by-water',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 30787f7259c4e6a08f691cc691f14fa0c8fe4b87 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 19:28:48 +0100
Subject: [PATCH 1478/2145] [cspan] correct the clip info extraction

---
 youtube_dl/extractor/cspan.py | 58 ++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index fbefd37d0..994e080d5 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -18,22 +18,21 @@ class CSpanIE(InfoExtractor):
     IE_DESC = 'C-SPAN'
     _TESTS = [{
         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
-        'md5': '8e44ce11f0f725527daccc453f553eb0',
+        'md5': '067803f994e049b455a58b16e5aab442',
         'info_dict': {
             'id': '315139',
             'ext': 'mp4',
             'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
-            'description': 'Attorney General Eric Holder spoke to reporters following the Supreme Court decision in Shelby County v. Holder in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced until Congress established new guidelines for review.',
+            'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
         },
         'skip': 'Regularly fails on travis, for unknown reasons',
     }, {
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
-        # For whatever reason, the served video alternates between
-        # two different ones
+        'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2',
         'info_dict': {
-            'id': '340723',
+            'id': 'c4486943',
             'ext': 'mp4',
-            'title': 'International Health Care Models',
+            'title': 'CSPAN - International Health Care Models',
             'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
         }
     }, {
@@ -44,7 +43,7 @@ class CSpanIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'General Motors Ignition Switch Recall',
             'duration': 14848,
-            'description': 'md5:70c7c3b8fa63fa60d42772440596034c'
+            'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
         },
     }, {
         # Video from senate.gov
@@ -57,36 +56,33 @@ class CSpanIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('id')
-        webpage = self._download_webpage(url, page_id)
-        video_id = self._search_regex(r'progid=\'?([0-9]+)\'?>', webpage, 'video id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
+        if matches:
+            video_type, video_id = matches.groups()
+            if video_type == 'prog':
+                video_type = 'program'
+        else:
+            senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+            if senate_isvp_url:
+                title = self._og_search_title(webpage)
+                surl = smuggle_url(senate_isvp_url, {'force_title': title})
+                return self.url_result(surl, 'SenateISVP', video_id, title)
 
-        description = self._html_search_regex(
-            [
-                # The full description
-                r'<div class=\'expandable\'>(.*?)<a href=\'#\'',
-                # If the description is small enough the other div is not
-                # present, otherwise this is a stripped version
-                r'<p class=\'initial\'>(.*?)</p>'
-            ],
-            webpage, 'description', flags=re.DOTALL, default=None)
-
-        info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
-        data = self._download_json(info_url, video_id)
+        data = self._download_json(
+            'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+            video_id)
 
         doc = self._download_xml(
-            'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
+            'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
             video_id)
 
+        description = self._html_search_meta('description', webpage)
+
         title = find_xpath_attr(doc, './/string', 'name', 'title').text
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
-        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
-        if senate_isvp_url:
-            surl = smuggle_url(senate_isvp_url, {'force_title': title})
-            return self.url_result(surl, 'SenateISVP', video_id, title)
-
         files = data['video']['files']
         try:
             capfile = data['video']['capfile']['#text']
@@ -112,12 +108,12 @@ class CSpanIE(InfoExtractor):
 
         if len(entries) == 1:
             entry = dict(entries[0])
-            entry['id'] = video_id
+            entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
             return entry
         else:
             return {
                 '_type': 'playlist',
                 'entries': entries,
                 'title': title,
-                'id': video_id,
+                'id': 'c' + video_id if video_type == 'clip' else video_id,
             }

From ef5acfe32de4c995625f9800cfe0776237961436 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 1 Sep 2015 23:05:19 +0100
Subject: [PATCH 1479/2145] [limelight] Add new extractor

---
 youtube_dl/extractor/__init__.py  |   5 +
 youtube_dl/extractor/limelight.py | 176 ++++++++++++++++++++++++++++++
 2 files changed, 181 insertions(+)
 create mode 100644 youtube_dl/extractor/limelight.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 495a18c17..20cc3660c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -295,6 +295,11 @@ from .lifenews import (
     LifeNewsIE,
     LifeEmbedIE,
 )
+from .limelight import (
+    LimeLightMediaIE,
+    LimeLightChannelIE,
+    LimeLightChannelListIE,
+)
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
new file mode 100644
index 000000000..dcfc215c7
--- /dev/null
+++ b/youtube_dl/extractor/limelight.py
@@ -0,0 +1,176 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    determine_ext,
+)
+
+
+class LimeLightBaseIE(InfoExtractor):
+
+    def get_playlist_service(self, id, method):
+        return self._download_json(self.PLAYLIST_SERVICE_URL % (id, method), id)
+
+    def get_api(self, orgId, id, method):
+        return self._download_json(self.API_URL % (orgId, id, method), id)
+
+    def process_data(self, mobileUrls, streams, properties):
+        video_id = properties['media_id']
+        formats = []
+
+        for mobileUrl in mobileUrls:
+            if '.m3u8' in mobileUrl['mobileUrl']:
+                formats.extend(self._extract_m3u8_formats(mobileUrl['mobileUrl'], video_id))
+            else:
+                formats.append({'url': mobileUrl['mobileUrl']})
+
+        for stream in streams:
+            if '.f4m' in stream['url']:
+                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+            else:
+                fmt = {
+                    'url': stream.get('url'),
+                    'abr': stream.get('audioBitRate'),
+                    'vbr': stream.get('videoBitRate'),
+                    'fps': stream.get('videoFrameRate'),
+                    'width': stream.get('videoWidthInPixels'),
+                    'height': stream.get('videoHeightInPixels'),
+                    'ext': determine_ext(stream.get('url'))
+                }
+                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream['url'])
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                    })
+                formats.append(fmt)
+
+        self._sort_formats(formats)
+
+        title = properties['title']
+        description = properties.get('description')
+        timestamp = properties.get('create_date')
+        duration = int_or_none(properties.get('duration_in_milliseconds'))
+        filesize = properties.get('total_storage_in_bytes')
+        categories = [properties.get('category')]
+        thumbnails = [{
+            'url': thumbnail.get('url'),
+            'width': int_or_none(thumbnail.get('width')),
+            'height': int_or_none(thumbnail.get('height')),
+        } for thumbnail in properties.get('thumbnails')]
+        subtitles = {caption.get('language_code'): [{'url': caption.get('url')}] for caption in properties.get('captions')}
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize': filesize,
+            'categories': categories,
+            'thumbnails': thumbnails,
+            'subtitles': subtitles,
+        }
+
+
+class LimeLightMediaIE(LimeLightBaseIE):
+    IE_NAME = 'limelight'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*mediaId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
+        'md5': '3213605088be599705677ef785db6972',
+        'info_dict': {
+            'id': '3ffd040b522b4485b6d84effc750cd86',
+            'ext': 'mp4',
+            'title': 'HaP and the HB Prince Trailer',
+            'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
+            'thumbnail': 're:^https?://.*\.jpeg$',
+            'duration': 144230,
+            'timestamp': 1244136834,
+            "upload_date": "20090604",
+        }
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/media/%s/%s'
+    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/media/%s/%s.json'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        mobile_json_data = self.get_playlist_service(video_id, 'getMobilePlaylistByMediaId')
+        pc_json_data = self.get_playlist_service(video_id, 'getPlaylistByMediaId')
+        properties = self.get_api(pc_json_data['orgId'], video_id, 'properties')
+
+        return self.process_data(mobile_json_data['mediaList'][0]['mobileUrls'], pc_json_data['playlistItems'][0]['streams'], properties)
+
+
+class LimeLightChannelIE(LimeLightBaseIE):
+    IE_NAME = 'limelight:channel'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
+        'info_dict': {
+            'id': 'ab6a524c379342f9b23642917020c082',
+            'title': 'Javascript Sample Code',
+        },
+        'playlist_mincount': 3,
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel/%s/%s'
+    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/channels/%s/%s.json'
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        mobile_json_data = self.get_playlist_service(channel_id, 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1')
+        pc_json_data = self.get_playlist_service(channel_id, 'getPlaylistByChannelId')
+        medias = self.get_api(pc_json_data['orgId'], channel_id, 'media')
+
+        entries = []
+        for i in range(len(medias['media_list'])):
+            entries.append(self.process_data(mobile_json_data['mediaList'][i]['mobileUrls'], pc_json_data['playlistItems'][i]['streams'], medias['media_list'][i]))
+
+        return {
+            'id': channel_id,
+            'title': pc_json_data['title'],
+            'entries': entries,
+            '_type': 'playlist',
+        }
+
+
+class LimeLightChannelListIE(LimeLightBaseIE):
+    IE_NAME = 'limelight:channel_list'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelListId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
+        'info_dict': {
+            'id': '301b117890c4465c8179ede21fd92e2b',
+            'title': 'Website - Hero Player',
+        },
+        'playlist_mincount': 2,
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel_list/%s/%s'
+
+    def _real_extract(self, url):
+        channel_list_id = self._match_id(url)
+
+        json_data = self.get_playlist_service(channel_list_id, 'getMobileChannelListById')
+
+        entries = []
+        for channel in json_data['channelList']:
+            entries.append({
+                'url': 'http://link.videoplatform.limelight.com/media/?channelId=%s' % channel['id'],
+                '_type': 'url',
+                'ie_key': 'LimeLightChannel',
+            })
+
+        return {
+            'id': channel_list_id,
+            'title': json_data['title'],
+            'entries': entries,
+            '_type': 'playlist',
+        }

From 4bba371644818d79b5f8481b5b31c53ea8ecbcc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:33:42 +0600
Subject: [PATCH 1480/2145] [YoutubeDL] Autocalculate ext for subtitles when
 missing

---
 youtube_dl/YoutubeDL.py        | 11 +++++++++--
 youtube_dl/extractor/common.py |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d65253882..adf70d658 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1232,13 +1232,20 @@ class YoutubeDL(object):
             except (ValueError, OverflowError, OSError):
                 pass
 
+        subtitles = info_dict.get('subtitles')
+        if subtitles:
+            for _, subtitle in subtitles.items():
+                for subtitle_format in subtitle:
+                    if 'ext' not in subtitle_format:
+                        subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
+
         if self.params.get('listsubtitles', False):
             if 'automatic_captions' in info_dict:
                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
-            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
+            self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
             return
         info_dict['requested_subtitles'] = self.process_subtitles(
-            info_dict['id'], info_dict.get('subtitles'),
+            info_dict['id'], subtitles,
             info_dict.get('automatic_captions'))
 
         # We now pick which formats have to be downloaded
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4fe2307cd..dbae75406 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -165,6 +165,7 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A URL pointing to the subtitles file
+                    "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
     duration:       Length of the video in seconds, as an integer.

From d7fc56318b72607758e7484c22076ec2999f10b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:41:57 +0600
Subject: [PATCH 1481/2145] [limelight] Fix python 2.6, simplify, make more
 robust (Closes #6734)

---
 youtube_dl/extractor/__init__.py  |   6 +-
 youtube_dl/extractor/limelight.py | 185 ++++++++++++++++++------------
 2 files changed, 113 insertions(+), 78 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 20cc3660c..3ace1cc2c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -296,9 +296,9 @@ from .lifenews import (
     LifeEmbedIE,
 )
 from .limelight import (
-    LimeLightMediaIE,
-    LimeLightChannelIE,
-    LimeLightChannelListIE,
+    LimelightMediaIE,
+    LimelightChannelIE,
+    LimelightChannelListIE,
 )
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index dcfc215c7..599d8413d 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -5,65 +5,105 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
     determine_ext,
+    float_or_none,
+    int_or_none,
 )
 
 
-class LimeLightBaseIE(InfoExtractor):
+class LimelightBaseIE(InfoExtractor):
+    _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
+    _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
 
-    def get_playlist_service(self, id, method):
-        return self._download_json(self.PLAYLIST_SERVICE_URL % (id, method), id)
+    def _call_playlist_service(self, item_id, method, fatal=True):
+        return self._download_json(
+            self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
+            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)
 
-    def get_api(self, orgId, id, method):
-        return self._download_json(self.API_URL % (orgId, id, method), id)
+    def _call_api(self, organization_id, item_id, method):
+        return self._download_json(
+            self._API_URL % (organization_id, self._API_PATH, item_id, method),
+            item_id, 'Downloading API %s JSON' % method)
 
-    def process_data(self, mobileUrls, streams, properties):
+    def _extract(self, item_id, pc_method, mobile_method, meta_method):
+        pc = self._call_playlist_service(item_id, pc_method)
+        metadata = self._call_api(pc['orgId'], item_id, meta_method)
+        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
+        return pc, mobile, metadata
+
+    def _extract_info(self, streams, mobile_urls, properties):
         video_id = properties['media_id']
         formats = []
 
-        for mobileUrl in mobileUrls:
-            if '.m3u8' in mobileUrl['mobileUrl']:
-                formats.extend(self._extract_m3u8_formats(mobileUrl['mobileUrl'], video_id))
-            else:
-                formats.append({'url': mobileUrl['mobileUrl']})
-
         for stream in streams:
-            if '.f4m' in stream['url']:
-                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            if '.f4m' in stream_url:
+                formats.extend(self._extract_f4m_formats(stream_url, video_id))
             else:
                 fmt = {
-                    'url': stream.get('url'),
-                    'abr': stream.get('audioBitRate'),
-                    'vbr': stream.get('videoBitRate'),
-                    'fps': stream.get('videoFrameRate'),
-                    'width': stream.get('videoWidthInPixels'),
-                    'height': stream.get('videoHeightInPixels'),
-                    'ext': determine_ext(stream.get('url'))
+                    'url': stream_url,
+                    'abr': float_or_none(stream.get('audioBitRate')),
+                    'vbr': float_or_none(stream.get('videoBitRate')),
+                    'fps': float_or_none(stream.get('videoFrameRate')),
+                    'width': int_or_none(stream.get('videoWidthInPixels')),
+                    'height': int_or_none(stream.get('videoHeightInPixels')),
+                    'ext': determine_ext(stream_url)
                 }
-                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream['url'])
+                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                 if rtmp:
+                    format_id = 'rtmp'
+                    if stream.get('videoBitRate'):
+                        format_id += '-%d' % int_or_none(stream['videoBitRate'])
                     fmt.update({
                         'url': rtmp.group('url'),
                         'play_path': rtmp.group('playpath'),
                         'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': format_id,
                     })
                 formats.append(fmt)
 
+        for mobile_url in mobile_urls:
+            media_url = mobile_url.get('mobileUrl')
+            if not media_url:
+                continue
+            format_id = mobile_url.get('targetMediaPlatform')
+            if determine_ext(media_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    preference=-1, m3u8_id=format_id))
+            else:
+                formats.append({
+                    'url': media_url,
+                    'format_id': format_id,
+                    'preference': -1,
+                })
+
         self._sort_formats(formats)
 
         title = properties['title']
         description = properties.get('description')
-        timestamp = properties.get('create_date')
-        duration = int_or_none(properties.get('duration_in_milliseconds'))
-        filesize = properties.get('total_storage_in_bytes')
+        timestamp = int_or_none(properties.get('publish_date') or properties.get('create_date'))
+        duration = float_or_none(properties.get('duration_in_milliseconds'), 1000)
+        filesize = int_or_none(properties.get('total_storage_in_bytes'))
         categories = [properties.get('category')]
+        tags = properties.get('tags', [])
         thumbnails = [{
-            'url': thumbnail.get('url'),
+            'url': thumbnail['url'],
             'width': int_or_none(thumbnail.get('width')),
             'height': int_or_none(thumbnail.get('height')),
-        } for thumbnail in properties.get('thumbnails')]
-        subtitles = {caption.get('language_code'): [{'url': caption.get('url')}] for caption in properties.get('captions')}
+        } for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
+
+        subtitles = {}
+        for caption in properties.get('captions', {}):
+            lang = caption.get('language_code')
+            subtitles_url = caption.get('url')
+            if lang and subtitles_url:
+                subtitles[lang] = [{
+                    'url': subtitles_url,
+                }]
 
         return {
             'id': video_id,
@@ -74,44 +114,50 @@ class LimeLightBaseIE(InfoExtractor):
             'duration': duration,
             'filesize': filesize,
             'categories': categories,
+            'tags': tags,
             'thumbnails': thumbnails,
             'subtitles': subtitles,
         }
 
 
-class LimeLightMediaIE(LimeLightBaseIE):
+class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*mediaId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
-        'md5': '3213605088be599705677ef785db6972',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'HaP and the HB Prince Trailer',
             'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
             'thumbnail': 're:^https?://.*\.jpeg$',
-            'duration': 144230,
+            'duration': 144.23,
             'timestamp': 1244136834,
-            "upload_date": "20090604",
-        }
+            'upload_date': '20090604',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/media/%s/%s'
-    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/media/%s/%s.json'
+    _PLAYLIST_SERVICE_PATH = 'media'
+    _API_PATH = 'media'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        mobile_json_data = self.get_playlist_service(video_id, 'getMobilePlaylistByMediaId')
-        pc_json_data = self.get_playlist_service(video_id, 'getPlaylistByMediaId')
-        properties = self.get_api(pc_json_data['orgId'], video_id, 'properties')
+        pc, mobile, metadata = self._extract(
+            video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')
 
-        return self.process_data(mobile_json_data['mediaList'][0]['mobileUrls'], pc_json_data['playlistItems'][0]['streams'], properties)
+        return self._extract_info(
+            pc['playlistItems'][0].get('streams', []),
+            mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
+            metadata)
 
 
-class LimeLightChannelIE(LimeLightBaseIE):
+class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel:|http://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {
@@ -120,31 +166,29 @@ class LimeLightChannelIE(LimeLightBaseIE):
         },
         'playlist_mincount': 3,
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel/%s/%s'
-    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/channels/%s/%s.json'
+    _PLAYLIST_SERVICE_PATH = 'channel'
+    _API_PATH = 'channels'
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        mobile_json_data = self.get_playlist_service(channel_id, 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1')
-        pc_json_data = self.get_playlist_service(channel_id, 'getPlaylistByChannelId')
-        medias = self.get_api(pc_json_data['orgId'], channel_id, 'media')
+        pc, mobile, medias = self._extract(
+            channel_id, 'getPlaylistByChannelId',
+            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')
 
-        entries = []
-        for i in range(len(medias['media_list'])):
-            entries.append(self.process_data(mobile_json_data['mediaList'][i]['mobileUrls'], pc_json_data['playlistItems'][i]['streams'], medias['media_list'][i]))
+        entries = [
+            self._extract_info(
+                pc['playlistItems'][i].get('streams', []),
+                mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
+                medias['media_list'][i])
+            for i in range(len(medias['media_list']))]
 
-        return {
-            'id': channel_id,
-            'title': pc_json_data['title'],
-            'entries': entries,
-            '_type': 'playlist',
-        }
+        return self.playlist_result(entries, channel_id, pc['title'])
 
 
-class LimeLightChannelListIE(LimeLightBaseIE):
+class LimelightChannelListIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel_list'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelListId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel_list:|http://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
         'info_dict': {
@@ -153,24 +197,15 @@ class LimeLightChannelListIE(LimeLightBaseIE):
         },
         'playlist_mincount': 2,
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel_list/%s/%s'
+    _PLAYLIST_SERVICE_PATH = 'channel_list'
 
     def _real_extract(self, url):
         channel_list_id = self._match_id(url)
 
-        json_data = self.get_playlist_service(channel_list_id, 'getMobileChannelListById')
+        channel_list = self._call_playlist_service(channel_list_id, 'getMobileChannelListById')
 
-        entries = []
-        for channel in json_data['channelList']:
-            entries.append({
-                'url': 'http://link.videoplatform.limelight.com/media/?channelId=%s' % channel['id'],
-                '_type': 'url',
-                'ie_key': 'LimeLightChannel',
-            })
+        entries = [
+            self.url_result('limelight:channel:%s' % channel['id'], 'LimelightChannel')
+            for channel in channel_list['channelList']]
 
-        return {
-            'id': channel_list_id,
-            'title': json_data['title'],
-            'entries': entries,
-            '_type': 'playlist',
-        }
+        return self.playlist_result(entries, channel_list_id, channel_list['title'])

From 9c544e2537abda1d65e96f2b33a79984f3ab7c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:48:44 +0600
Subject: [PATCH 1482/2145] [limelight] Add test video with subtitles

---
 youtube_dl/extractor/limelight.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 599d8413d..fb03dd527 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,13 +123,13 @@ class LimelightBaseIE(InfoExtractor):
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
     _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
             'ext': 'flv',
             'title': 'HaP and the HB Prince Trailer',
-            'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
+            'description': 'md5:8005b944181778e313d95c1237ddb640',
             'thumbnail': 're:^https?://.*\.jpeg$',
             'duration': 144.23,
             'timestamp': 1244136834,
@@ -139,7 +139,25 @@ class LimelightMediaIE(LimelightBaseIE):
             # rtmp download
             'skip_download': True,
         },
-    }
+    }, {
+        # video with subtitles
+        'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
+        'info_dict': {
+            'id': 'a3e00274d4564ec4a9b29b9466432335',
+            'ext': 'flv',
+            'title': '3Play Media Overview Video',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpeg$',
+            'duration': 78.101,
+            'timestamp': 1338929955,
+            'upload_date': '20120605',
+            'subtitles': 'mincount:9',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
     _PLAYLIST_SERVICE_PATH = 'media'
     _API_PATH = 'media'
 

From 0659dfccfea9df3206c476e83a2b090456c25a83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:13:13 +0600
Subject: [PATCH 1483/2145] [pbs] Improve player regex (Closes #7059)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 683c81de3..0bca3152b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -167,7 +167,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
+                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].+?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 96229998c29705c8ee4230915ec7ff050bcfecf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:19:47 +0600
Subject: [PATCH 1484/2145] [pbs] Allow empty attribute in player regex

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0bca3152b..66b3dda47 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -167,7 +167,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].+?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
+                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].*?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 90ab741e909c949039e31805da04f5e546a1a8c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:37:49 +0600
Subject: [PATCH 1485/2145] [pbs] Add test for #7059

---
 youtube_dl/extractor/pbs.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 66b3dda47..6923c6094 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -134,6 +134,24 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            # Video embedded in iframe containing angle brackets as attribute's value (e.g.
+            # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
+            # https://github.com/rg3/youtube-dl/issues/7059)
+            'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
+            'info_dict': {
+                'id': '2365546844',
+                'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
+                'ext': 'mp4',
+                'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
+                'description': 'md5:61db2ddf27c9912f09c241014b118ed1',
+                'duration': 1480,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
 

From e759a00119768862e63dbda33522f2399f1f43a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 20:21:53 +0600
Subject: [PATCH 1486/2145] [appletrailers] Quotes consistency

---
 youtube_dl/extractor/appletrailers.py | 68 +++++++++++++--------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 576f03b5b..f68dc3236 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -13,53 +13,53 @@ from ..utils import (
 class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
-        "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
         'info_dict': {
             'id': 'manofsteel',
         },
-        "playlist": [
+        'playlist': [
             {
-                "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
-                "info_dict": {
-                    "id": "manofsteel-trailer4",
-                    "ext": "mov",
-                    "duration": 111,
-                    "title": "Trailer 4",
-                    "upload_date": "20130523",
-                    "uploader_id": "wb",
+                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
+                'info_dict': {
+                    'id': 'manofsteel-trailer4',
+                    'ext': 'mov',
+                    'duration': 111,
+                    'title': 'Trailer 4',
+                    'upload_date': '20130523',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "b8017b7131b721fb4e8d6f49e1df908c",
-                "info_dict": {
-                    "id": "manofsteel-trailer3",
-                    "ext": "mov",
-                    "duration": 182,
-                    "title": "Trailer 3",
-                    "upload_date": "20130417",
-                    "uploader_id": "wb",
+                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
+                'info_dict': {
+                    'id': 'manofsteel-trailer3',
+                    'ext': 'mov',
+                    'duration': 182,
+                    'title': 'Trailer 3',
+                    'upload_date': '20130417',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "d0f1e1150989b9924679b441f3404d48",
-                "info_dict": {
-                    "id": "manofsteel-trailer",
-                    "ext": "mov",
-                    "duration": 148,
-                    "title": "Trailer",
-                    "upload_date": "20121212",
-                    "uploader_id": "wb",
+                'md5': 'd0f1e1150989b9924679b441f3404d48',
+                'info_dict': {
+                    'id': 'manofsteel-trailer',
+                    'ext': 'mov',
+                    'duration': 148,
+                    'title': 'Trailer',
+                    'upload_date': '20121212',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "5fe08795b943eb2e757fa95cb6def1cb",
-                "info_dict": {
-                    "id": "manofsteel-teaser",
-                    "ext": "mov",
-                    "duration": 93,
-                    "title": "Teaser",
-                    "upload_date": "20120721",
-                    "uploader_id": "wb",
+                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
+                'info_dict': {
+                    'id': 'manofsteel-teaser',
+                    'ext': 'mov',
+                    'duration': 93,
+                    'title': 'Teaser',
+                    'upload_date': '20120721',
+                    'uploader_id': 'wb',
                 },
             },
         ]

From 2a27e66234ec0030d2ca6f1ddf229db7f7eb8ded Mon Sep 17 00:00:00 2001
From: Oli Allen <oli@oliallen.com>
Date: Thu, 1 Oct 2015 22:50:12 +0100
Subject: [PATCH 1487/2145] [tumblr] Added support for HD video where available
 (#7036)

[tumblr] Replaced test URL for HD video as old one lead to 404

[tumblr] Don't make assumptions about video resolution, cleaner handling of no HD version available

[tumblr] Removed extraneous resolution key in HD video tests
---
 youtube_dl/extractor/tumblr.py | 58 ++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 3d3b635e4..71cced562 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -28,6 +28,32 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
+        'md5': '99a84522f60972bf064a0b80f87bcbb5',
+        'info_dict': {
+            'id': '130323439814',
+            'ext': 'mp4',
+            'title': 'HD Video Testing \u2014 Test description for my HD video',
+            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'format': 'sd',
+        },
+    }, {
+        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
+        'md5': '7ae503065ad150122dc3089f8cf1546c',
+        'info_dict': {
+            'id': '130323439814',
+            'ext': 'mp4',
+            'title': 'HD Video Testing \u2014 Test description for my HD video',
+            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'format': 'hd',
+        },
     }, {
         'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
         'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
@@ -57,6 +83,8 @@ class TumblrIE(InfoExtractor):
         video_id = m_url.group('id')
         blog = m_url.group('blog_name')
 
+        video_urls = []
+
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -68,8 +96,32 @@ class TumblrIE(InfoExtractor):
 
         iframe = self._download_webpage(iframe_url, video_id,
                                         'Downloading iframe page')
-        video_url = self._search_regex(r'<source src="([^"]+)"',
-                                       iframe, 'video url')
+
+        sd_video_url = self._search_regex(r'<source src="([^"]+)"',
+                                          iframe, 'sd video url')
+        resolution_id = sd_video_url.split("/")[-1] + 'p'
+        if len(resolution_id) != 4:
+            resolution_id = None
+        video_urls.append({
+            'ext': 'mp4',
+            'format_id': 'sd',
+            'url': sd_video_url,
+            'resolution': resolution_id,
+        })
+
+        hd_video_url = self._search_regex(r'hdUrl":"([^"]+)"', iframe,
+                                          'hd video url', default=None)
+        if hd_video_url:
+            hd_video_url = hd_video_url.replace("\\", "")
+            resolution_id = hd_video_url.split("/")[-1] + 'p'
+            if len(resolution_id) != 4:
+                resolution_id = None
+            video_urls.append({
+                'ext': 'mp4',
+                'format_id': 'hd',
+                'url': hd_video_url,
+                'resolution': resolution_id,
+            })
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -79,7 +131,7 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': video_urls,
             'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),

From 140ac7396542d92a8ddd53be6c35c7a79db16180 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:53:01 +0600
Subject: [PATCH 1488/2145] [tumblr] Simplify and extract duration

---
 youtube_dl/extractor/tumblr.py | 60 ++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 71cced562..449ba29fa 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class TumblrIE(InfoExtractor):
@@ -83,8 +84,6 @@ class TumblrIE(InfoExtractor):
         video_id = m_url.group('id')
         blog = m_url.group('blog_name')
 
-        video_urls = []
-
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -94,34 +93,38 @@ class TumblrIE(InfoExtractor):
         if iframe_url is None:
             return self.url_result(urlh.geturl(), 'Generic')
 
-        iframe = self._download_webpage(iframe_url, video_id,
-                                        'Downloading iframe page')
+        iframe = self._download_webpage(iframe_url, video_id, 'Downloading iframe page')
 
-        sd_video_url = self._search_regex(r'<source src="([^"]+)"',
-                                          iframe, 'sd video url')
-        resolution_id = sd_video_url.split("/")[-1] + 'p'
-        if len(resolution_id) != 4:
-            resolution_id = None
-        video_urls.append({
+        duration = None
+        sources = []
+
+        sd_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', iframe,
+            'sd video url', default=None, group='url')
+        if sd_url:
+            sources.append((sd_url, 'sd'))
+
+        options = self._parse_json(
+            self._search_regex(
+                r'data-crt-options=(["\'])(?P<options>.+?)\1', iframe,
+                'hd video url', default='', group='options'),
+            video_id, fatal=False)
+        if options:
+            duration = int_or_none(options.get('duration'))
+            hd_url = options.get('hdUrl')
+            if hd_url:
+                sources.append((hd_url, 'hd'))
+
+        formats = [{
+            'url': video_url,
             'ext': 'mp4',
-            'format_id': 'sd',
-            'url': sd_video_url,
-            'resolution': resolution_id,
-        })
+            'format_id': format_id,
+            'height': int_or_none(self._search_regex(
+                r'/(\d{3,4})$', video_url, 'height', default=None)),
+            'quality': quality,
+        } for quality, (video_url, format_id) in enumerate(sources)]
 
-        hd_video_url = self._search_regex(r'hdUrl":"([^"]+)"', iframe,
-                                          'hd video url', default=None)
-        if hd_video_url:
-            hd_video_url = hd_video_url.replace("\\", "")
-            resolution_id = hd_video_url.split("/")[-1] + 'p'
-            if len(resolution_id) != 4:
-                resolution_id = None
-            video_urls.append({
-                'ext': 'mp4',
-                'format_id': 'hd',
-                'url': hd_video_url,
-                'resolution': resolution_id,
-            })
+        self._sort_formats(formats)
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -131,9 +134,10 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'formats': video_urls,
             'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
+            'duration': duration,
+            'formats': formats,
         }

From 5d84b79a3002925084690e17669738bf1ea711f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:53:59 +0600
Subject: [PATCH 1489/2145] [tumblr] Remove redundant test

---
 youtube_dl/extractor/tumblr.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 449ba29fa..9f270318b 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -29,19 +29,6 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
-    }, {
-        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
-        'md5': '99a84522f60972bf064a0b80f87bcbb5',
-        'info_dict': {
-            'id': '130323439814',
-            'ext': 'mp4',
-            'title': 'HD Video Testing \u2014 Test description for my HD video',
-            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
-            'thumbnail': 're:http://.*\.jpg',
-        },
-        'params': {
-            'format': 'sd',
-        },
     }, {
         'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
         'md5': '7ae503065ad150122dc3089f8cf1546c',

From 88c86d211bd2f542625d3b91ac4b15e62995345d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:54:54 +0600
Subject: [PATCH 1490/2145] [tumblr] Add missing fields for vidme test

---
 youtube_dl/extractor/tumblr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 9f270318b..cb91d08eb 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -51,6 +51,9 @@ class TumblrIE(InfoExtractor):
             'title': 'naked smoking & stretching',
             'upload_date': '20150506',
             'timestamp': 1430931613,
+            'age_limit': 18,
+            'uploader_id': '1638622',
+            'uploader': 'naked-yogi',
         },
         'add_ie': ['Vidme'],
     }, {

From 7fd4ed9939d7c467c61663cad7189ee8ad27c89b Mon Sep 17 00:00:00 2001
From: David Rabinowitz <drabinowitz@agtinternational.com>
Date: Thu, 1 Oct 2015 12:07:27 +0300
Subject: [PATCH 1491/2145] Fixed the ustream extractor to use the current
 ustream API

---
 youtube_dl/extractor/ustream.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index c39c278ab..18add908e 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -52,17 +52,12 @@ class UstreamIE(InfoExtractor):
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        params = self._download_json(
-            'http://cdngw.ustream.tv/rgwjson/Viewer.getVideo/' + json.dumps({
-                'brandId': 1,
-                'videoId': int(video_id),
-                'autoplay': False,
-            }), video_id)
+        params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
 
         if 'error' in params:
             raise ExtractorError(params['error']['message'], expected=True)
 
-        video_url = params['flv']
+        video_url = params['video']['media_urls']['flv']
 
         webpage = self._download_webpage(url, video_id)
 

From 5820c4a29ea3781c47f6249c179a081e28eeeb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:30:38 +0600
Subject: [PATCH 1492/2145] [ustream] Switch extraction to api

---
 youtube_dl/extractor/ustream.py | 69 ++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 18add908e..3065c9f31 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -7,7 +7,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+)
 
 
 class UstreamIE(InfoExtractor):
@@ -54,46 +58,47 @@ class UstreamIE(InfoExtractor):
 
         params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
 
-        if 'error' in params:
-            raise ExtractorError(params['error']['message'], expected=True)
+        error = params.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        video_url = params['video']['media_urls']['flv']
+        video = params['video']
 
-        webpage = self._download_webpage(url, video_id)
+        formats = [{
+            'id': format_id,
+            'url': video_url,
+            'ext': format_id,
+        } for format_id, video_url in video['media_urls'].items()]
+        self._sort_formats(formats)
 
-        self.report_extraction(video_id)
+        title = video['title']
+        description = video.get('description')
+        timestamp = int_or_none(video.get('created_at'))
+        duration = float_or_none(video.get('length'))
+        filesize = float_or_none(video.get('file_size'))
+        view_count = int_or_none(video.get('views'))
 
-        video_title = self._html_search_regex(r'data-title="(?P<title>.+)"',
-                                              webpage, 'title', default=None)
+        uploader = video.get('owner', {}).get('username')
+        uploader_id = video.get('owner', {}).get('id')
 
-        if not video_title:
-            try:
-                video_title = params['moduleConfig']['meta']['title']
-            except KeyError:
-                pass
-
-        if not video_title:
-            video_title = 'Ustream video ' + video_id
-
-        uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
-                                           webpage, 'uploader', fatal=False, flags=re.DOTALL, default=None)
-
-        if not uploader:
-            try:
-                uploader = params['moduleConfig']['meta']['userName']
-            except KeyError:
-                uploader = None
-
-        thumbnail = self._html_search_regex(r'<link rel="image_src" href="(?P<thumb>.*?)"',
-                                            webpage, 'thumbnail', fatal=False)
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in video.get('thumbnail', {}).items()]
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'flv',
-            'title': video_title,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize': filesize,
+            'view_count': view_count,
             'uploader': uploader,
-            'thumbnail': thumbnail,
+            'uploader_id': uploader_id,
+            'formats': formats,
         }
 
 

From 4853eb63fec766c7154e9353e91e531a3f3b1d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:40:20 +0600
Subject: [PATCH 1493/2145] [ustream] Modernize

---
 youtube_dl/extractor/ustream.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 3065c9f31..9fdeb064d 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class UstreamIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<videoID>\d+)'
+    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
     IE_NAME = 'ustream'
     _TESTS = [{
         'url': 'http://www.ustream.tv/recorded/20274954',
@@ -41,22 +41,23 @@ class UstreamIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = m.group('id')
 
         # some sites use this embed format (see: http://github.com/rg3/youtube-dl/issues/2990)
         if m.group('type') == 'embed/recorded':
-            video_id = m.group('videoID')
+            video_id = m.group('id')
             desktop_url = 'http://www.ustream.tv/recorded/' + video_id
             return self.url_result(desktop_url, 'Ustream')
         if m.group('type') == 'embed':
-            video_id = m.group('videoID')
+            video_id = m.group('id')
             webpage = self._download_webpage(url, video_id)
             desktop_video_id = self._html_search_regex(
                 r'ContentVideoIds=\["([^"]*?)"\]', webpage, 'desktop_video_id')
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
+        params = self._download_json(
+            'https://api.ustream.tv/videos/%s.json' % video_id, video_id)
 
         error = params.get('error')
         if error:

From f2a7ed77ef76b8d564326f564afc880473fba7ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:44:36 +0600
Subject: [PATCH 1494/2145] [tumblr] Remove redundant field

---
 youtube_dl/extractor/tumblr.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index cb91d08eb..4f844706d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -124,7 +124,6 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),

From 0bf219889e39d4d7e75fdb59d7452b8e09f4ddab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:44:59 +0600
Subject: [PATCH 1495/2145] [ustream] Remove unused import

---
 youtube_dl/extractor/ustream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 9fdeb064d..7243d0eca 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor

From 41db73330835d6e8fe29ce18f869247a739467ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:48:47 +0600
Subject: [PATCH 1496/2145] [ustream] Move filesize

---
 youtube_dl/extractor/ustream.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 7243d0eca..888f39f7a 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -65,18 +65,20 @@ class UstreamIE(InfoExtractor):
 
         video = params['video']
 
+        title = video['title']
+        filesize = float_or_none(video.get('file_size'))
+
         formats = [{
             'id': format_id,
             'url': video_url,
             'ext': format_id,
+            'filesize': filesize,
         } for format_id, video_url in video['media_urls'].items()]
         self._sort_formats(formats)
 
-        title = video['title']
         description = video.get('description')
         timestamp = int_or_none(video.get('created_at'))
         duration = float_or_none(video.get('length'))
-        filesize = float_or_none(video.get('file_size'))
         view_count = int_or_none(video.get('views'))
 
         uploader = video.get('owner', {}).get('username')
@@ -94,7 +96,6 @@ class UstreamIE(InfoExtractor):
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,
-            'filesize': filesize,
             'view_count': view_count,
             'uploader': uploader,
             'uploader_id': uploader_id,

From dc5756fd7729f616185508fba296b330f0892c24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:51:04 +0600
Subject: [PATCH 1497/2145] [ustream] Fix typo

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 888f39f7a..a29d67e9f 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -69,7 +69,7 @@ class UstreamIE(InfoExtractor):
         filesize = float_or_none(video.get('file_size'))
 
         formats = [{
-            'id': format_id,
+            'id': video_id,
             'url': video_url,
             'ext': format_id,
             'filesize': filesize,

From 40fbb05e1c58625349160ec134343af789b803ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:52:51 +0600
Subject: [PATCH 1498/2145] [ustream] Fix tests

---
 youtube_dl/extractor/ustream.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index a29d67e9f..73b05ecab 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -22,8 +22,12 @@ class UstreamIE(InfoExtractor):
         'info_dict': {
             'id': '20274954',
             'ext': 'flv',
-            'uploader': 'Young Americans for Liberty',
             'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'description': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'timestamp': 1328577035,
+            'upload_date': '20120207',
+            'uploader': 'yaliberty',
+            'uploader_id': '6780869',
         },
     }, {
         # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
@@ -35,7 +39,8 @@ class UstreamIE(InfoExtractor):
             'ext': 'flv',
             'title': '-CG11- Canada Games Figure Skating',
             'uploader': 'sportscanadatv',
-        }
+        },
+        'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
     }]
 
     def _real_extract(self, url):

From fcc25462693fb0468d66b28664ba622f8f3bdb3d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 02:30:05 +0600
Subject: [PATCH 1499/2145] [README.md] Markdown improvements

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 80152071d..e7004c9fc 100644
--- a/README.md
+++ b/README.md
@@ -359,7 +359,7 @@ If you have installed youtube-dl with a package manager, pip, setup.py or a tarb
 
 By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
-### Can you please put the -b option back?
+### Can you please put the `-b` option back?
 
 Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
 
@@ -371,13 +371,13 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
 
-### I extracted a video URL with -g, but it does not play on another machine / in my webbrowser.
+### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
 It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
-Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using -g, your own downloader must support these as well.
+Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
 If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
@@ -643,15 +643,15 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 

From c4af7684d85a17441c5f6f0b6e9f8c470644fec0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 09:08:10 +0200
Subject: [PATCH 1500/2145] release 2015.10.06

---
 CONTRIBUTING.md        |   6 +-
 README.md              | 395 +++++++++++++++++++++++++++++------------
 docs/supportedsites.md |   4 +
 youtube_dl/version.py  |   2 +-
 4 files changed, 289 insertions(+), 118 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f8ab29631..57a94231d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -16,15 +16,15 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 
diff --git a/README.md b/README.md
index e7004c9fc..5ff8ca85e 100644
--- a/README.md
+++ b/README.md
@@ -49,110 +49,220 @@ which means you can modify it, redistribute it or use it however you like.
 # OPTIONS
     -h, --help                       Print this help text and exit
     --version                        Print program version and exit
-    -U, --update                     Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
-    -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
-    --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
+    -U, --update                     Update this program to latest version. Make
+                                     sure that you have sufficient permissions
+                                     (run with sudo if needed)
+    -i, --ignore-errors              Continue on download errors, for example to
+                                     skip unavailable videos in a playlist
+    --abort-on-error                 Abort downloading of further videos (in the
+                                     playlist or the command line) if an error
+                                     occurs
     --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors
-    --extractor-descriptions         Output descriptions of all supported extractors
-    --force-generic-extractor        Force extraction to use the generic extractor
-    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
-                                     Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
-                                     default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
-    --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
-                                     in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
-    --flat-playlist                  Do not extract the videos of a playlist, only list them.
+    --extractor-descriptions         Output descriptions of all supported
+                                     extractors
+    --force-generic-extractor        Force extraction to use the generic
+                                     extractor
+    --default-search PREFIX          Use this prefix for unqualified URLs. For
+                                     example "gvsearch2:" downloads two videos
+                                     from google videos for youtube-dl "large
+                                     apple". Use the value "auto" to let
+                                     youtube-dl guess ("auto_warning" to emit a
+                                     warning when guessing). "error" just throws
+                                     an error. The default value "fixup_error"
+                                     repairs broken URLs, but emits an error if
+                                     this is not possible instead of searching.
+    --ignore-config                  Do not read configuration files. When given
+                                     in the global configuration file /etc
+                                     /youtube-dl.conf: Do not read the user
+                                     configuration in ~/.config/youtube-
+                                     dl/config (%APPDATA%/youtube-dl/config.txt
+                                     on Windows)
+    --flat-playlist                  Do not extract the videos of a playlist,
+                                     only list them.
     --no-color                       Do not emit color codes in output
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
+                                     an empty string (--proxy "") for direct
+                                     connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
-    --source-address IP              Client-side IP address to bind to (experimental)
-    -4, --force-ipv4                 Make all connections via IPv4 (experimental)
-    -6, --force-ipv6                 Make all connections via IPv6 (experimental)
-    --cn-verification-proxy URL      Use this proxy to verify the IP address for some Chinese sites. The default proxy specified by --proxy (or none, if the options is
-                                     not present) is used for the actual downloading. (experimental)
+    --source-address IP              Client-side IP address to bind to
+                                     (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4
+                                     (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6
+                                     (experimental)
+    --cn-verification-proxy URL      Use this proxy to verify the IP address for
+                                     some Chinese sites. The default proxy
+                                     specified by --proxy (or none, if the
+                                     options is not present) is used for the
+                                     actual downloading. (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
     --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8"
-                                     if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
-                                     download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              Download only matching titles (regex or caseless sub-string)
-    --reject-title REGEX             Skip download for matching titles (regex or caseless sub-string)
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify
+                                     indices of the videos in the playlist
+                                     separated by commas like: "--playlist-items
+                                     1,2,5,8" if you want to download videos
+                                     indexed 1, 2, 5, 8 in the playlist. You can
+                                     specify range: "--playlist-items
+                                     1-3,7,10-13", it will download the videos
+                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
+    --match-title REGEX              Download only matching titles (regex or
+                                     caseless sub-string)
+    --reject-title REGEX             Skip download for matching titles (regex or
+                                     caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
-    --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
-    --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
+    --min-filesize SIZE              Do not download any videos smaller than
+                                     SIZE (e.g. 50k or 44.6m)
+    --max-filesize SIZE              Do not download any videos larger than SIZE
+                                     (e.g. 50k or 44.6m)
     --date DATE                      Download only videos uploaded in this date
-    --datebefore DATE                Download only videos uploaded on or before this date (i.e. inclusive)
-    --dateafter DATE                 Download only videos uploaded on or after this date (i.e. inclusive)
-    --min-views COUNT                Do not download any videos with less than COUNT views
-    --max-views COUNT                Do not download any videos with more than COUNT views
-    --match-filter FILTER            Generic video filter (experimental). Specify any key (see help for -o for a list of available keys) to match if the key is present,
-                                     !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
-                                     a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
-                                     operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
-                                     functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
+    --datebefore DATE                Download only videos uploaded on or before
+                                     this date (i.e. inclusive)
+    --dateafter DATE                 Download only videos uploaded on or after
+                                     this date (i.e. inclusive)
+    --min-views COUNT                Do not download any videos with less than
+                                     COUNT views
+    --max-views COUNT                Do not download any videos with more than
+                                     COUNT views
+    --match-filter FILTER            Generic video filter (experimental).
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    Download only the video, if the URL refers to a video and a playlist.
-    --yes-playlist                   Download the playlist, if the URL refers to a video and a playlist.
-    --age-limit YEARS                Download only videos suitable for the given age
-    --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
-    --include-ads                    Download advertisements as well (experimental)
+    --no-playlist                    Download only the video, if the URL refers
+                                     to a video and a playlist.
+    --yes-playlist                   Download the playlist, if the URL refers to
+                                     a video and a playlist.
+    --age-limit YEARS                Download only videos suitable for the given
+                                     age
+    --download-archive FILE          Download only videos not listed in the
+                                     archive file. Record the IDs of all
+                                     downloaded videos in it.
+    --include-ads                    Download advertisements as well
+                                     (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           Maximum download rate in bytes per second (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            Number of retries (default is 10), or "infinite".
-    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K) (default is 1024)
-    --no-resize-buffer               Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
+    -r, --rate-limit LIMIT           Maximum download rate in bytes per second
+                                     (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            Number of retries (default is 10), or
+                                     "infinite".
+    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
+                                     (default is 1024)
+    --no-resize-buffer               Do not automatically adjust the buffer
+                                     size. By default, the buffer size is
+                                     automatically resized from an initial value
+                                     of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
-    --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,axel,curl,httpie,wget
-    --external-downloader-args ARGS  Give these arguments to the external downloader
+    --xattr-set-filesize             Set file xattribute ytdl.filesize with
+                                     expected filesize (experimental)
+    --hls-prefer-native              Use the native HLS downloader instead of
+                                     ffmpeg (experimental)
+    --external-downloader COMMAND    Use the specified external downloader.
+                                     Currently supports
+                                     aria2c,axel,curl,httpie,wget
+    --external-downloader-args ARGS  Give these arguments to the external
+                                     downloader
 
 ## Filesystem Options:
-    -a, --batch-file FILE            File containing URLs to download ('-' for stdin)
+    -a, --batch-file FILE            File containing URLs to download ('-' for
+                                     stdin)
     --id                             Use only video ID in file name
-    -o, --output TEMPLATE            Output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
-                                     nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
-                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like YouTube's itags: "137"),
-                                     %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
-                                     %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
-                                     %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
-                                     %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
-                                     Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
-    --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
-    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000
-    -t, --title                      [deprecated] Use title in file name (default)
+    -o, --output TEMPLATE            Output filename template. Use %(title)s to
+                                     get the title, %(uploader)s for the
+                                     uploader name, %(uploader_id)s for the
+                                     uploader nickname if different,
+                                     %(autonumber)s to get an automatically
+                                     incremented number, %(ext)s for the
+                                     filename extension, %(format)s for the
+                                     format description (like "22 - 1280x720" or
+                                     "HD"), %(format_id)s for the unique id of
+                                     the format (like YouTube's itags: "137"),
+                                     %(upload_date)s for the upload date
+                                     (YYYYMMDD), %(extractor)s for the provider
+                                     (youtube, metacafe, etc), %(id)s for the
+                                     video id, %(playlist_title)s,
+                                     %(playlist_id)s, or %(playlist)s (=title if
+                                     present, ID otherwise) for the playlist the
+                                     video is in, %(playlist_index)s for the
+                                     position in the playlist. %(height)s and
+                                     %(width)s for the width and height of the
+                                     video format. %(resolution)s for a textual
+                                     description of the resolution of the video
+                                     format. %% for a literal percent. Use - to
+                                     output to stdout. Can also be used to
+                                     download to a different directory, for
+                                     example with -o '/my/downloads/%(uploader)s
+                                     /%(title)s-%(id)s.%(ext)s' .
+    --autonumber-size NUMBER         Specify the number of digits in
+                                     %(autonumber)s when it is present in output
+                                     filename template or --auto-number option
+                                     is given
+    --restrict-filenames             Restrict filenames to only ASCII
+                                     characters, and avoid "&" and spaces in
+                                     filenames
+    -A, --auto-number                [deprecated; use -o
+                                     "%(autonumber)s-%(title)s.%(ext)s" ] Number
+                                     downloaded files starting from 00000
+    -t, --title                      [deprecated] Use title in file name
+                                     (default)
     -l, --literal                    [deprecated] Alias of --title
     -w, --no-overwrites              Do not overwrite files
-    -c, --continue                   Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
-    --no-continue                    Do not resume partially downloaded files (restart from beginning)
-    --no-part                        Do not use .part files - write directly into output file
-    --no-mtime                       Do not use the Last-modified header to set the file modification time
-    --write-description              Write video description to a .description file
+    -c, --continue                   Force resume of partially downloaded files.
+                                     By default, youtube-dl will resume
+                                     downloads if possible.
+    --no-continue                    Do not resume partially downloaded files
+                                     (restart from beginning)
+    --no-part                        Do not use .part files - write directly
+                                     into output file
+    --no-mtime                       Do not use the Last-modified header to set
+                                     the file modification time
+    --write-description              Write video description to a .description
+                                     file
     --write-info-json                Write video metadata to a .info.json file
-    --write-annotations              Write video annotations to a .annotations.xml file
-    --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
-    --cookies FILE                   File to read cookies from and dump cookie jar in
-    --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
-                                     or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
-                                     change.
+    --write-annotations              Write video annotations to a
+                                     .annotations.xml file
+    --load-info FILE                 JSON file containing the video information
+                                     (created with the "--write-info-json"
+                                     option)
+    --cookies FILE                   File to read cookies from and dump cookie
+                                     jar in
+    --cache-dir DIR                  Location in the filesystem where youtube-dl
+                                     can store some downloaded information
+                                     permanently. By default $XDG_CACHE_HOME
+                                     /youtube-dl or ~/.cache/youtube-dl . At the
+                                     moment, only YouTube player files (for
+                                     videos with obfuscated signatures) are
+                                     cached, but that may change.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
     --write-thumbnail                Write thumbnail image to disk
     --write-all-thumbnails           Write all thumbnail image formats to disk
-    --list-thumbnails                Simulate and list all available thumbnail formats
+    --list-thumbnails                Simulate and list all available thumbnail
+                                     formats
 
 ## Verbosity / Simulation Options:
     -q, --quiet                      Activate quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   Do not download the video and do not write anything to disk
+    -s, --simulate                   Do not download the video and do not write
+                                     anything to disk
     --skip-download                  Do not download the video
     -g, --get-url                    Simulate, quiet but print URL
     -e, --get-title                  Simulate, quiet but print title
@@ -162,78 +272,135 @@ which means you can modify it, redistribute it or use it however you like.
     --get-duration                   Simulate, quiet but print video length
     --get-filename                   Simulate, quiet but print output filename
     --get-format                     Simulate, quiet but print output format
-    -j, --dump-json                  Simulate, quiet but print JSON information. See --output for a description of available keys.
-    -J, --dump-single-json           Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
-                                     information in a single line.
-    --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
+    -j, --dump-json                  Simulate, quiet but print JSON information.
+                                     See --output for a description of available
+                                     keys.
+    -J, --dump-single-json           Simulate, quiet but print JSON information
+                                     for each command-line argument. If the URL
+                                     refers to a playlist, dump the whole
+                                     playlist information in a single line.
+    --print-json                     Be quiet and print the video information as
+                                     JSON (video is still being downloaded).
     --newline                        Output progress bar as new lines
     --no-progress                    Do not print progress bar
     --console-title                  Display progress in console titlebar
     -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages encoded using base64 to debug problems (very verbose)
-    --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
+    --dump-pages                     Print downloaded pages encoded using base64
+                                     to debug problems (very verbose)
+    --write-pages                    Write downloaded intermediary pages to
+                                     files in the current directory to debug
+                                     problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging
-    --no-call-home                   Do NOT contact the youtube-dl server for debugging
+    --no-call-home                   Do NOT contact the youtube-dl server for
+                                     debugging
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
     --no-check-certificate           Suppress HTTPS certificate validation
-    --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
+    --prefer-insecure                Use an unencrypted connection to retrieve
+                                     information about the video. (Currently
+                                     supported only for YouTube)
     --user-agent UA                  Specify a custom user agent
-    --referer URL                    Specify a custom referer, use if the video access is restricted to one domain
-    --add-header FIELD:VALUE         Specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
-    --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
-    --sleep-interval SECONDS         Number of seconds to sleep before each download.
+    --referer URL                    Specify a custom referer, use if the video
+                                     access is restricted to one domain
+    --add-header FIELD:VALUE         Specify a custom HTTP header and its value,
+                                     separated by a colon ':'. You can use this
+                                     option multiple times
+    --bidi-workaround                Work around terminals that lack
+                                     bidirectional text support. Requires bidiv
+                                     or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each
+                                     download.
 
 ## Video Format Options:
-    -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
+    -f, --format FORMAT              Video format code, see the "FORMAT
+                                     SELECTION" for all the info
     --all-formats                    Download all available video formats
-    --prefer-free-formats            Prefer free video formats unless a specific one is requested
+    --prefer-free-formats            Prefer free video formats unless a specific
+                                     one is requested
     -F, --list-formats               List all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifests and related data on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv. Ignored if no
-                                     merge is required
+    --youtube-skip-dash-manifest     Do not download the DASH manifests and
+                                     related data on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g.
+                                     bestvideo+bestaudio), output to given
+                                     container format. One of mkv, mp4, ogg,
+                                     webm, flv. Ignored if no merge is required
 
 ## Subtitle Options:
     --write-sub                      Write subtitle file
-    --write-auto-sub                 Write automatic subtitle file (YouTube only)
-    --all-subs                       Download all the available subtitles of the video
+    --write-auto-sub                 Write automatic subtitle file (YouTube
+                                     only)
+    --all-subs                       Download all the available subtitles of the
+                                     video
     --list-subs                      List all available subtitles for the video
-    --sub-format FORMAT              Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"
-    --sub-lang LANGS                 Languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
+    --sub-format FORMAT              Subtitle format, accepts formats
+                                     preference, for example: "srt" or
+                                     "ass/srt/best"
+    --sub-lang LANGS                 Languages of the subtitles to download
+                                     (optional) separated by commas, use IETF
+                                     language tags like 'en,pt'
 
 ## Authentication Options:
     -u, --username USERNAME          Login with this account ID
-    -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
+    -p, --password PASSWORD          Account password. If this option is left
+                                     out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
     --video-password PASSWORD        Video password (vimeo, smotri, youku)
 
 ## Post-processing Options:
-    -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
-                                     5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)
+    -x, --extract-audio              Convert video files to audio-only files
+                                     (requires ffmpeg or avconv and ffprobe or
+                                     avprobe)
+    --audio-format FORMAT            Specify audio format: "best", "aac",
+                                     "vorbis", "mp3", "m4a", "opus", or "wav";
+                                     "best" by default
+    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
+                                     a value between 0 (better) and 9 (worse)
+                                     for VBR or a specific bitrate like 128K
+                                     (default 5)
+    --recode-video FORMAT            Encode the video to another format if
+                                     necessary (currently supported:
+                                     mp4|flv|ogg|webm|mkv|avi)
     --postprocessor-args ARGS        Give these arguments to the postprocessor
-    -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
-    --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
+    -k, --keep-video                 Keep the video file on disk after the post-
+                                     processing; the video is erased by default
+    --no-post-overwrites             Do not overwrite post-processed files; the
+                                     post-processed files are overwritten by
+                                     default
+    --embed-subs                     Embed subtitles in the video (only for mkv
+                                     and mp4 videos)
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
-    --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
-                                     parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
-                                     %(title)s" matches a title like "Coldplay - Paradise"
-    --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the default;
-                                     fix file if we can, warn otherwise)
-    --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
-    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
-    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.
-    --exec CMD                       Execute a command on the file after downloading, similar to find's -exec syntax. Example: --exec 'adb push {} /sdcard/Music/ && rm
-                                     {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format (currently supported: srt|ass|vtt)
+    --metadata-from-title FORMAT     Parse additional metadata like song title /
+                                     artist from the video title. The format
+                                     syntax is the same as --output, the parsed
+                                     parameters replace existing values.
+                                     Additional templates: %(album)s,
+                                     %(artist)s. Example: --metadata-from-title
+                                     "%(artist)s - %(title)s" matches a title
+                                     like "Coldplay - Paradise"
+    --xattrs                         Write metadata to the video file's xattrs
+                                     (using dublin core and xdg standards)
+    --fixup POLICY                   Automatically correct known faults of the
+                                     file. One of never (do nothing), warn (only
+                                     emit a warning), detect_or_warn (the
+                                     default; fix file if we can, warn
+                                     otherwise)
+    --prefer-avconv                  Prefer avconv over ffmpeg for running the
+                                     postprocessors (default)
+    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
+                                     postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
+                                     either the path to the binary or its
+                                     containing directory.
+    --exec CMD                       Execute a command on the file after
+                                     downloading, similar to find's -exec
+                                     syntax. Example: --exec 'adb push {}
+                                     /sdcard/Music/ && rm {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format
+                                     (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fa83b68ad..5beb2ecd4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -150,6 +150,7 @@
  - **Escapist**
  - **ESPN** (Currently broken)
  - **EsriVideo**
+ - **Europa**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -263,6 +264,9 @@
  - **Libsyn**
  - **life:embed**
  - **lifenews**: LIFE | NEWS
+ - **limelight**
+ - **limelight:channel**
+ - **limelight:channel_list**
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8f0977849..46c3356e9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.28'
+__version__ = '2015.10.06'

From 4810c48d6d7e950c2cb1203f4d07bea1ba02c1e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 14:28:14 +0200
Subject: [PATCH 1501/2145] [compat] Do not compare None <= 0

The result is meaningless (and it emits a warning in cpython2 when called with -3), so handle None before making integer comparisons.
---
 youtube_dl/compat.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c36c9c23f..1ba4ab78c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -417,18 +417,18 @@ else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
-        columns = compat_getenv('COLUMNS', None)
+        columns = compat_getenv('COLUMNS')
         if columns:
             columns = int(columns)
         else:
             columns = None
-        lines = compat_getenv('LINES', None)
+        lines = compat_getenv('LINES')
         if lines:
             lines = int(lines)
         else:
             lines = None
 
-        if columns <= 0 or lines <= 0:
+        if columns is None or lines is None or columns <= 0 or lines <= 0:
             try:
                 sp = subprocess.Popen(
                     ['stty', 'size'],
@@ -438,9 +438,9 @@ else:
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
 
-            if columns <= 0:
+            if columns is None or columns <= 0:
                 columns = _columns
-            if lines <= 0:
+            if lines is None or lines <= 0:
                 lines = _lines
         return _terminal_size(columns, lines)
 

From 86be82610c35a684bee97b22c8d9a2a83bab1bba Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 17:43:50 +0200
Subject: [PATCH 1502/2145] release 2015.10.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 46c3356e9..4fce70fa6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06'
+__version__ = '2015.10.06.1'

From f2dbc54066f56a98c689099b920e6a596d4ffdfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 22:02:28 +0600
Subject: [PATCH 1503/2145] [compat] Fix wrong lines/columns order

stty size is rows x columns
---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1ba4ab78c..192e1c515 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -434,7 +434,7 @@ else:
                     ['stty', 'size'],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 out, err = sp.communicate()
-                _columns, _lines = map(int, out.split())
+                _lines, _columns = map(int, out.split())
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
 

From f648e682a7a82f71a278ec92159ac5d343a4b3eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 22:58:18 +0600
Subject: [PATCH 1504/2145] [bandcamp] Prepend download URL with scheme when
 necessary (Closes #7077)

---
 youtube_dl/extractor/bandcamp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 505877b77..a27f3e748 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -93,8 +93,8 @@ class BandcampIE(InfoExtractor):
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         # in the "download_url" key
-        final_url = self._search_regex(
-            r'"retry_url":"(.*?)"', final_url_webpage, 'final video URL')
+        final_url = self._proto_relative_url(self._search_regex(
+            r'"retry_url":"(.+?)"', final_url_webpage, 'final video URL'), 'http:')
 
         return {
             'id': video_id,

From 83a56686944225137ce646748b8e940a21584941 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 23:08:28 +0600
Subject: [PATCH 1505/2145] [canalplus] Extend video id regex (Closes #7076)

---
 youtube_dl/extractor/canalplus.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 57e0cda2c..c0ca3a0fa 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -78,7 +78,8 @@ class CanalplusIE(InfoExtractor):
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
-                r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
+                [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
+                 webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')

From fc10824cb67feb837ea57d60decc293b7b719cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Oct 2015 02:43:12 +0600
Subject: [PATCH 1506/2145] [canalplus] PEP 8

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index c0ca3a0fa..004372f8d 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -79,7 +79,7 @@ class CanalplusIE(InfoExtractor):
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
                 [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
-                 webpage, 'video id', group='id')
+                webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')

From 945e5c56e3ccd45cdcc3ee45b9ffcbad7e614f90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 23:46:16 +0200
Subject: [PATCH 1507/2145] release 2015.10.06.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4fce70fa6..112c78835 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06.1'
+__version__ = '2015.10.06.2'

From 139f27827e1d771aba5cf7f1473129073686f5ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 7 Oct 2015 06:53:19 +0100
Subject: [PATCH 1508/2145] [nba] skip Legacy Video Files

---
 youtube_dl/extractor/nba.py | 35 +++++++++++------------------------
 1 file changed, 11 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 3d38d080e..73116c7c6 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -38,11 +38,6 @@ class NBAIE(InfoExtractor):
         }
     }]
 
-    _BASE_PATHS = {
-        'turner': 'http://nba.cdn.turner.com/nba/big',
-        'akamai': 'http://nbavod-f.akamaihd.net',
-    }
-
     _QUALITIES = {
         '420mp4': {
             'width': 400,
@@ -54,55 +49,50 @@ class NBAIE(InfoExtractor):
             'height': 234,
             'preference': 2,
         },
-        '556': {
-            'width': 416,
-            'height': 234,
-            'preference': 3,
-        },
         '480x320_910': {
             'width': 480,
             'height': 320,
-            'preference': 4,
+            'preference': 3,
         },
         'nba_576x324': {
             'width': 576,
             'height': 324,
-            'preference': 5,
+            'preference': 4,
         },
         'nba_640x360': {
             'width': 640,
             'height': 360,
-            'preference': 6,
+            'preference': 5,
         },
         '640x360_664b': {
             'width': 640,
             'height': 360,
-            'preference': 7,
+            'preference': 6,
         },
         '640x360_664m': {
             'width': 640,
             'height': 360,
-            'preference': 8,
+            'preference': 7,
         },
         '768x432_996': {
             'width': 768,
             'height': 432,
-            'preference': 9,
+            'preference': 8,
         },
         '768x432_1404': {
             'width': 768,
             'height': 432,
-            'preference': 10,
+            'preference': 9,
         },
         '960x540_2104': {
             'width': 960,
             'height': 540,
-            'preference': 11,
+            'preference': 10,
         },
         '1280x720_3072': {
             'width': 1280,
             'height': 720,
-            'preference': 12,
+            'preference': 11,
         },
     }
 
@@ -127,11 +117,8 @@ class NBAIE(InfoExtractor):
         formats = []
         for video_file in video_info.find('files').iter('file'):
             video_url = video_file.text
-            if not video_url.startswith('http://'):
-                if video_url.endswith('.m3u8') or video_url.endswith('.f4m'):
-                    video_url = self._BASE_PATHS['akamai'] + video_url
-                else:
-                    video_url = self._BASE_PATHS['turner'] + video_url
+            if video_url.startswith('/'):
+                continue
             if video_url.endswith('.m3u8'):
                 formats.extend(self._extract_m3u8_formats(video_url, video_id))
             elif video_url.endswith('.f4m'):

From ecf6de5b02ad3996f770efd33f9b400d04ac8a85 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 7 Oct 2015 07:09:45 +0100
Subject: [PATCH 1509/2145] [nba] extract width,height and bitrate from format
 key

---
 youtube_dl/extractor/nba.py | 68 ++++---------------------------------
 1 file changed, 6 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 73116c7c6..ea1482fc8 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
@@ -38,64 +40,6 @@ class NBAIE(InfoExtractor):
         }
     }]
 
-    _QUALITIES = {
-        '420mp4': {
-            'width': 400,
-            'height': 224,
-            'preference': 1,
-        },
-        '416x234': {
-            'width': 416,
-            'height': 234,
-            'preference': 2,
-        },
-        '480x320_910': {
-            'width': 480,
-            'height': 320,
-            'preference': 3,
-        },
-        'nba_576x324': {
-            'width': 576,
-            'height': 324,
-            'preference': 4,
-        },
-        'nba_640x360': {
-            'width': 640,
-            'height': 360,
-            'preference': 5,
-        },
-        '640x360_664b': {
-            'width': 640,
-            'height': 360,
-            'preference': 6,
-        },
-        '640x360_664m': {
-            'width': 640,
-            'height': 360,
-            'preference': 7,
-        },
-        '768x432_996': {
-            'width': 768,
-            'height': 432,
-            'preference': 8,
-        },
-        '768x432_1404': {
-            'width': 768,
-            'height': 432,
-            'preference': 9,
-        },
-        '960x540_2104': {
-            'width': 960,
-            'height': 540,
-            'preference': 10,
-        },
-        '1280x720_3072': {
-            'width': 1280,
-            'height': 720,
-            'preference': 11,
-        },
-    }
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id)
@@ -125,13 +69,13 @@ class NBAIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id))
             else:
                 key = video_file.attrib.get('bitrate')
-                quality = self._QUALITIES[key]
+                width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups()
                 formats.append({
                     'format_id': key,
                     'url': video_url,
-                    'width': quality['width'],
-                    'height': quality['height'],
-                    'preference': quality['preference'],
+                    'width': int_or_none(width),
+                    'height': int_or_none(height),
+                    'tbr': int_or_none(bitrate),
                 })
         self._sort_formats(formats)
 

From bd5376c182bd5ee2103aec21144286dbefbeb797 Mon Sep 17 00:00:00 2001
From: Tom Gijselinck <tomgijselinck@gmail.com>
Date: Wed, 7 Oct 2015 10:32:44 +0200
Subject: [PATCH 1510/2145] Fix typos and improve grammar and spelling

---
 README.md | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 5ff8ca85e..cf4aebf3d 100644
--- a/README.md
+++ b/README.md
@@ -404,18 +404,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, system wide configuration file is located at `/etc/youtube-dl.conf` and user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configration file youtube-dl will always extract the audio, not copy the mtime and use proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable configuration file for a particular youtube-dl run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file ###
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in shell command history. You can achieve this using [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -429,13 +429,13 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or place it in [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
-On Windows you may also need to setup `%HOME%` environment variable manually.
+On Windows you may also need to setup the `%HOME%` environment variable manually.
 
 # OUTPUT TEMPLATE
 
-The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:
+The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
  - `id`: The sequence will be replaced by the video identifier.
  - `url`: The sequence will be replaced by the video URL.
@@ -463,18 +463,18 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
+By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 
-Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
+Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`. They accept dates in two formats:
 
  - Absolute dates: Dates in the format `YYYYMMDD`.
  - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
@@ -488,7 +488,7 @@ $ youtube-dl --dateafter now-6months
 # Download only the videos uploaded on January 1, 1970
 $ youtube-dl --date 19700101
 
-$ # will only download the videos uploaded in the 200x decade
+$ # Download only the videos uploaded in the 200x decade
 $ youtube-dl --dateafter 20000101 --datebefore 20091231
 ```
 
@@ -500,7 +500,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distributions serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -558,7 +558,7 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 ### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -618,9 +618,9 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How do I pass cookies to youtube-dl?
 
-Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that cookies file must be in Mozilla/Netscape format and the first line of cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in cookies file and convert newlines if necessary to correspond your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
@@ -720,7 +720,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             }
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:

From 6a11bb77baf9f70da76f2595b74061b31223d4ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 7 Oct 2015 12:17:32 +0100
Subject: [PATCH 1511/2145] [nba] add support for team subsites

---
 youtube_dl/extractor/nba.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index ea1482fc8..a0cc58c12 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -10,13 +10,13 @@ from ..utils import (
 
 
 class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video/(?P<id>[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)?video/(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
-        'md5': '9d902940d2a127af3f7f9d2f3dc79c96',
+        'md5': '9e7729d3010a9c71506fd1248f74e4f4',
         'info_dict': {
             'id': '0021200253-okc-bkn-recap',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
@@ -27,7 +27,7 @@ class NBAIE(InfoExtractor):
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
     },{
-        'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
             'id': '0041400301-cle-atl-recap',
@@ -41,8 +41,8 @@ class NBAIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id)
+        path, video_id = re.match(self._VALID_URL, url).groups()
+        video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
         video_id = video_info.find('slug').text
         title = video_info.find('headline').text
         description = video_info.find('description').text
@@ -64,9 +64,9 @@ class NBAIE(InfoExtractor):
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls'))
             elif video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id))
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds'))
             else:
                 key = video_file.attrib.get('bitrate')
                 width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups()

From db7a28eccb743db4e546a12fb78ae5dc40ef44d9 Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Thu, 8 Oct 2015 09:56:39 +0200
Subject: [PATCH 1512/2145] FIX: nowtv

now the download works for me
thx to http://board.gulli.com/thread/1251646-videos-von-webseiten-runterladen-sammelthread-alle-fragen-hier-rein-/?p=14980081#post14980081
---
 youtube_dl/extractor/nowtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index c8257719f..b0bdffc4e 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -167,8 +167,8 @@ class NowTVIE(InfoExtractor):
                 'app': app,
                 'play_path': 'mp4:%s' % play_path,
                 'ext': 'flv',
-                'page_url': url,
-                'player_url': 'http://rtl-now.rtl.de/includes/nc_player.swf',
+                'page_url': 'http://rtlnow.rtl.de',
+                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
                 'tbr': int_or_none(item.get('bitrate')),
             })
         self._sort_formats(formats)

From b90b0c4ffa6f84a2ea5556d4df99de5f8ef2c7dd Mon Sep 17 00:00:00 2001
From: kitty <magicvidyakitty@gmail.com>
Date: Thu, 8 Oct 2015 06:31:23 -0700
Subject: [PATCH 1513/2145] Fixed 4tube.com extractor to pull metadata from
 associated Javascript and not the HTML of the desired page.

---
 youtube_dl/extractor/fourtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 3bb4f6239..226ee67f0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -45,11 +45,9 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
-            webpage, 'uploader id')
+            r'<a class="img-avatar" href="[^"]+/users/([^/"]+)" title="Go to [^"]+ page">', webpage, 'uploader id')
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
-            webpage, 'uploader')
+            r'<a class="img-avatar" href="[^"]+/users/[^/"]+" title="Go to ([^"]+) page">', webpage, 'uploader')
 
         categories_html = self._search_regex(
             r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',
@@ -68,9 +66,12 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
+        player_url = self._search_regex(r'<script id="playerembed" src="([^"]+)">',webpage,'player javascript')
+        player_js = self._download_webpage(player_url,video_id,'Downloading player Javascript')
+
         params_js = self._search_regex(
             r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
-            webpage, 'initialization parameters'
+            player_js, 'initialization parameters'
         )
         params = self._parse_json('[%s]' % params_js, video_id)
         media_id = params[0]

From 96c48553317cf28ea92d7feaa5701e3ebd93ea25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:21:07 +0600
Subject: [PATCH 1514/2145] [4tube] Style and make more robust

---
 youtube_dl/extractor/fourtube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 226ee67f0..406387e57 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -66,8 +66,11 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        player_url = self._search_regex(r'<script id="playerembed" src="([^"]+)">',webpage,'player javascript')
-        player_js = self._download_webpage(player_url,video_id,'Downloading player Javascript')
+        player_js = self._download_webpage(
+            self._search_regex(
+                r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
+                webpage, 'player JS', group='url'),
+            video_id, 'Downloading player JS')
 
         params_js = self._search_regex(
             r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',

From c677e49bd11c3dfada05de9fc1d4de04a577d70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:22:08 +0600
Subject: [PATCH 1515/2145] [4tube] Revert uploader regexes and make non fatal

---
 youtube_dl/extractor/fourtube.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 406387e57..cd7668f18 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -45,9 +45,11 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/users/([^/"]+)" title="Go to [^"]+ page">', webpage, 'uploader id')
+            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            webpage, 'uploader id', fatal=False)
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/users/[^/"]+" title="Go to ([^"]+) page">', webpage, 'uploader')
+            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            webpage, 'uploader', fatal=False)
 
         categories_html = self._search_regex(
             r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',

From 4356d907c1afae09ff67a17c90b53be466b13623 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:49:56 +0600
Subject: [PATCH 1516/2145] [4tube] Try extracting sources from the webpage
 before fetching player.js (Closes #7103)

---
 youtube_dl/extractor/fourtube.py | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index cd7668f18..fb6d108c0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -68,19 +68,24 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        player_js = self._download_webpage(
-            self._search_regex(
-                r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
-                webpage, 'player JS', group='url'),
-            video_id, 'Downloading player JS')
-
-        params_js = self._search_regex(
-            r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
-            player_js, 'initialization parameters'
-        )
-        params = self._parse_json('[%s]' % params_js, video_id)
-        media_id = params[0]
-        sources = ['%s' % p for p in params[2]]
+        media_id = self._search_regex(
+            r'<button[^>]+data-id=(["\'])(?P<id>\d+)\1[^>]+data-quality=', webpage,
+            'media id', default=None, group='id')
+        sources = [
+            quality
+            for _, quality in re.findall(r'<button[^>]+data-quality=(["\'])(.+?)\1', webpage)]
+        if not (media_id and sources):
+            player_js = self._download_webpage(
+                self._search_regex(
+                    r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
+                    webpage, 'player JS', group='url'),
+                video_id, 'Downloading player JS')
+            params_js = self._search_regex(
+                r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
+                player_js, 'initialization parameters')
+            params = self._parse_json('[%s]' % params_js, video_id)
+            media_id = params[0]
+            sources = ['%s' % p for p in params[2]]
 
         token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
             media_id, '+'.join(sources))

From 7faf7e752342b21772c8647902bbf4f13cc57956 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 8 Oct 2015 18:39:24 +0100
Subject: [PATCH 1517/2145] [pbs] detect errors from http error code

---
 youtube_dl/extractor/pbs.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6923c6094..82218b933 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -39,6 +39,7 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+            'skip': 'Expired',
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -108,12 +109,12 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
             'info_dict': {
-                'id': '2280706814',
+                'id': '2276541483',
                 'display_id': 'player',
                 'ext': 'mp4',
-                'title': 'American Experience - Death and the Civil War',
+                'title': 'American Experience - Death and the Civil War, Chapter 1',
                 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
-                'duration': 6705,
+                'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {
@@ -134,6 +135,7 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+            'skip': 'Expired',
         },
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
@@ -231,13 +233,18 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                if redirect_info['http_code'] == 403:
-                    message = (
-                        'The video is not available in your region due to '
-                        'right restrictions')
+                http_code = redirect_info['http_code']
+                if http_code == 403:
+                    message = 'We\'re sorry, but this video is not available in your region due to right restrictions.'
+                elif http_code == 101:
+                    message = 'We\'re sorry, but this video is not yet available.'
+                elif http_code == 404:
+                    message = 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.'
+                elif http_code == 410:
+                    message = 'This video has expired and is no longer available for online streaming.'
                 else:
                     message = redirect_info['message']
-                raise ExtractorError(message, expected=True)
+                raise ExtractorError('PBS said: %s' % message, expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 0553d0ee40b6ecd62ed25ba8999aa2e4f7641590 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 8 Oct 2015 18:57:57 +0100
Subject: [PATCH 1518/2145] [pbs] place errors into a dict

---
 youtube_dl/extractor/pbs.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 82218b933..814e97f48 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -39,7 +39,6 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
-            'skip': 'Expired',
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -156,6 +155,12 @@ class PBSIE(InfoExtractor):
             },
         }
     ]
+    _ERRORS = {
+        101: 'We\'re sorry, but this video is not yet available.',
+        403: 'We\'re sorry, but this video is not available in your region due to right restrictions.',
+        404: 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.',
+        410: 'This video has expired and is no longer available for online streaming.',
+    }
 
     def _extract_webpage(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -233,18 +238,7 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                http_code = redirect_info['http_code']
-                if http_code == 403:
-                    message = 'We\'re sorry, but this video is not available in your region due to right restrictions.'
-                elif http_code == 101:
-                    message = 'We\'re sorry, but this video is not yet available.'
-                elif http_code == 404:
-                    message = 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.'
-                elif http_code == 410:
-                    message = 'This video has expired and is no longer available for online streaming.'
-                else:
-                    message = redirect_info['message']
-                raise ExtractorError('PBS said: %s' % message, expected=True)
+                raise ExtractorError('PBS said: %s' % self._ERRORS.get(redirect_info['http_code'], redirect_info['message']), expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 9d5fb3b58d95fbd8c28d9c0eaf4a652660324d9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Oct 2015 00:09:10 +0600
Subject: [PATCH 1519/2145] [pbs] Carry long line

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 814e97f48..3448736a2 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -238,7 +238,11 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                raise ExtractorError('PBS said: %s' % self._ERRORS.get(redirect_info['http_code'], redirect_info['message']), expected=True)
+                raise ExtractorError(
+                    '%s said: %s' % (
+                        self.IE_NAME,
+                        self._ERRORS.get(redirect_info['http_code'], redirect_info['message'])),
+                    expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 54a5428518adc4ddca085bdac471bc4f286024e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Oct 2015 03:54:49 +0600
Subject: [PATCH 1520/2145] [dailymotion] Update player v5 regex (Closes #7107)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 2d90b2224..80a05cfee 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -119,7 +119,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(
-            r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+            [r'buildPlayer\(({.+?})\);', r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From 57935b2564c082b90a60468d4c844b219118886a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 9 Oct 2015 14:11:00 +0800
Subject: [PATCH 1521/2145] [extractor/common] Allow HTML5 unquoted attribute
 values

Fixes #7108

HTML5 allows unquoted attribute values. See the "Unquoted attribute value
syntax" section [1] for more information

[1] http://www.w3.org/TR/html5/syntax.html
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dbae75406..242618c58 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -646,7 +646,7 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
-        property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
+        property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
             template % (property_re, content_re),

From ef47b2c15f3fa9d9d491090f2ea46d2bd9967d21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Oct 2015 09:09:22 +0200
Subject: [PATCH 1522/2145] release 2015.10.09

---
 CONTRIBUTING.md       | 2 +-
 youtube_dl/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 57a94231d..32c2fd84c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -124,7 +124,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             }
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 112c78835..faae8a2d7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06.2'
+__version__ = '2015.10.09'

From 4180a3d8b7c40792b5371ca8804c1dad8fabd56d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 01:44:33 +0600
Subject: [PATCH 1523/2145] [extractor/common] Allow quoteless content
 attribute in og regexes (Closes #7115)

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 242618c58..0082a4c84 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -645,7 +645,7 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
+        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
         property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [

From 47c165c3a9f40dd9a175b11f29f9e2002fdda8a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 05:56:01 +0600
Subject: [PATCH 1524/2145] [vimeo] Fix authentication (Closes #7110)

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 50df79ca1..7dd52627d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -40,6 +40,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
         token = self._extract_xsrft(webpage)
+        vuid = self._search_regex(
+            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
+            webpage, 'vuid', group='vuid')
         data = urlencode_postdata({
             'action': 'login',
             'email': username,
@@ -49,6 +52,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         })
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_request.add_header('Cookie', 'vuid=%s' % vuid)
         login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 

From 6a959f2e5266dcc6037a33589eb34fad9190c63e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 10 Oct 2015 15:03:01 +0800
Subject: [PATCH 1525/2145] [iqiyi] Update enc_key

---
 youtube_dl/extractor/iqiyi.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ce1ab3820..0e53cb154 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -191,7 +191,7 @@ class IqiyiIE(InfoExtractor):
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': self.md5_text((enc_key + tail)[1:64:2] + tail),
+            'enc': self.md5_text(enc_key + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
@@ -205,7 +205,9 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        enc_key = 'eac64f22daf001da6ba9aa8da4d501508bbe90a4d4091fea3b0582a85b38c2cc'  # last update at 2015-09-23-23 for Zombie::bite
+        # last update at 2015-10-10 for Zombie::bite
+        # '7239670519b6ac209a0bee4ef0446a6b24894b8ac2751506e42116212a0d0272e505'[2:66][1::2]
+        enc_key = '97596c0abee04ab49ba25564161ad225'
         return enc_key
 
     def _real_extract(self, url):

From dab062fb6ecd48e0c243a6d030d89b44cd44bd84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:34:06 +0600
Subject: [PATCH 1526/2145] [bbc] Add support for videos in news articles
 embedded with data-playable

---
 youtube_dl/extractor/bbc.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index cc2f6fed2..b2b39ff21 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    unescapeHTML,
 )
 from ..compat import compat_HTTPError
 
@@ -682,6 +683,21 @@ class BBCIE(BBCCoUkIE):
             [r'data-video-player-vpid="([\da-z]{8})"',
              r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
             webpage, 'vpid', default=None)
+
+        duration = None
+        if not programme_id:
+            # single video in news article embedded with data-playable (e.g.
+            # http://www.bbc.com/news/world-us-canada-34473351)
+            data_playable = self._parse_json(
+                unescapeHTML(self._search_regex(
+                    r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
+                programme_id, fatal=False)
+            if data_playable:
+                items = data_playable.get('settings', {}).get('playlistObject', {}).get('items')
+                if items and isinstance(items, list):
+                    duration = int_or_none(items[0].get('duration'))
+                    programme_id = items[0].get('vpid')
+
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             self._sort_formats(formats)
@@ -699,6 +715,7 @@ class BBCIE(BBCCoUkIE):
                 'title': title,
                 'description': description,
                 'timestamp': timestamp,
+                'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
             }

From de66571371e2a9705ecd3aed903880c4846d04ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:40:56 +0600
Subject: [PATCH 1527/2145] [bbc] Support multiple videos in articles embedded
 with playlist.sxml

---
 youtube_dl/extractor/bbc.py | 38 ++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b2b39ff21..930637cd7 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -660,23 +660,27 @@ class BBCIE(BBCCoUkIE):
              r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
             webpage, 'date', default=None))
 
-        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
-        playlist = self._search_regex(
-            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
-            webpage, 'playlist', default=None)
-        if playlist:
-            programme_id, title, description, duration, formats, subtitles = \
-                self._process_legacy_playlist_url(playlist, playlist_id)
-            self._sort_formats(formats)
-            return {
-                'id': programme_id,
-                'title': title,
-                'description': description,
-                'duration': duration,
-                'timestamp': timestamp,
-                'formats': formats,
-                'subtitles': subtitles,
-            }
+        # article with multiple videos embedded with playlist.sxml (e.g.
+        # http://www.bbc.com/sport/0/football/34475836)
+        playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
+        if playlists:
+            entries = []
+            for playlist in playlists:
+                programme_id, title, description, duration, formats, subtitles = \
+                    self._process_legacy_playlist_url(playlist, playlist_id)
+                self._sort_formats(formats)
+                entries.append({
+                    'id': programme_id,
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'timestamp': timestamp,
+                    'formats': formats,
+                    'subtitles': subtitles,
+                })
+            playlist_title = self._og_search_title(webpage)
+            playlist_description = self._og_search_description(webpage)
+            return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(

From b5d48cb1ef741ef999d038a3bc80943b1ae37a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:55:46 +0600
Subject: [PATCH 1528/2145] [bbc] Add test for atricle with multiple videos
 embedded with playlist.sxml

---
 youtube_dl/extractor/bbc.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 930637cd7..ffe2afa96 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -608,6 +608,14 @@ class BBCIE(BBCCoUkIE):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        # article with multiple videos embedded with playlist.sxml
+        'url': 'http://www.bbc.com/sport/0/football/34475836',
+        'info_dict': {
+            'id': '34475836',
+            'title': 'What Liverpool can expect from Klopp',
+        },
+        'playlist_count': 3,
     }, {
         # single video with playlist URL from weather section
         'url': 'http://www.bbc.com/weather/features/33601775',

From 975977860d1880bbf5e9996ad77fb7f67622ccbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:56:07 +0600
Subject: [PATCH 1529/2145] [bbc] Make playlist title optional

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ffe2afa96..f943b5fd1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -687,7 +687,7 @@ class BBCIE(BBCCoUkIE):
                     'subtitles': subtitles,
                 })
             playlist_title = self._og_search_title(webpage)
-            playlist_description = self._og_search_description(webpage)
+            playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)

From baf39a1aa8af38ca295e5efdad277ddee5b11eb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 21:32:27 +0600
Subject: [PATCH 1530/2145] [bbc] Add one more scenario for data-playable
 embeds

---
 youtube_dl/extractor/bbc.py | 53 ++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f943b5fd1..df61f0157 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -657,6 +657,20 @@ class BBCIE(BBCCoUkIE):
 
         return [], []
 
+    def _extract_from_playlist_sxml(self, url, playlist_id, timestamp):
+        programme_id, title, description, duration, formats, subtitles = \
+            self._process_legacy_playlist_url(url, playlist_id)
+        self._sort_formats(formats)
+        return {
+            'id': programme_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
@@ -672,20 +686,9 @@ class BBCIE(BBCCoUkIE):
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
         if playlists:
-            entries = []
-            for playlist in playlists:
-                programme_id, title, description, duration, formats, subtitles = \
-                    self._process_legacy_playlist_url(playlist, playlist_id)
-                self._sort_formats(formats)
-                entries.append({
-                    'id': programme_id,
-                    'title': title,
-                    'description': description,
-                    'duration': duration,
-                    'timestamp': timestamp,
-                    'formats': formats,
-                    'subtitles': subtitles,
-                })
+            entries = [
+                self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
+                for playlist_url in playlists]
             playlist_title = self._og_search_title(webpage)
             playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
@@ -705,10 +708,24 @@ class BBCIE(BBCCoUkIE):
                     r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
                 programme_id, fatal=False)
             if data_playable:
-                items = data_playable.get('settings', {}).get('playlistObject', {}).get('items')
-                if items and isinstance(items, list):
-                    duration = int_or_none(items[0].get('duration'))
-                    programme_id = items[0].get('vpid')
+                # data-playable has video vpid in settings.playlistObject.items (e.g.
+                # http://www.bbc.com/news/world-us-canada-34473351)
+                settings = data_playable.get('settings', {})
+                if settings:
+                    playlist_object = settings.get('playlistObject', {})
+                    if playlist_object:
+                        items = playlist_object.get('items')
+                        if items and isinstance(items, list):
+                            duration = int_or_none(items[0].get('duration'))
+                            programme_id = items[0].get('vpid')
+                if not programme_id:
+                    # data-playable has no vpid but has a playlist.sxml URLs
+                    # in otherSettings.playlist (e.g.
+                    # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
+                    playlist = data_playable.get('otherSettings', {}).get('playlist', {})
+                    if playlist:
+                        return self._extract_from_playlist_sxml(
+                            playlist.get('progressiveDownloadUrl'), playlist_id, timestamp)
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)

From 87cc0fbd18345117926d27db7312cba663d0f5ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 21:35:20 +0600
Subject: [PATCH 1531/2145] [bbc] Make summary optional in legacy playlist

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index df61f0157..911ccd757 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -401,7 +401,8 @@ class BBCCoUkIE(InfoExtractor):
             if kind != 'programme' and kind != 'radioProgramme':
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+            description_el = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary')
+            description = description_el.text if description_el else None
 
             def get_programme_id(item):
                 def get_from_attributes(item):

From 97ae4d166c43b7e5068a7563bc5fbce385d86c2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ph=E1=BA=A1m=20Ng=E1=BB=8Dc=20Quang=20Nam?=
 <nampnq@gmail.com>
Date: Sat, 10 Oct 2015 22:07:06 +0700
Subject: [PATCH 1532/2145] [zingmp3:album] Add support for playlists

Update for work with playlist, it same album but different url
Ex:
http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html
http://mp3.zing.vn/album/Duong-Hong-Loan-apollobee/IWCAACCB.html
---
 youtube_dl/extractor/zingmp3.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 7dc1e2f2b..42ac124f0 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -85,16 +85,22 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
 
 
 class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/album/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [{
-        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-        'info_dict': {
-            '_type': 'playlist',
-            'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+    _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
+    _TESTS = [
+        {
+            'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+            'info_dict': {
+                '_type': 'playlist',
+                'id': 'ZWZBWDAF',
+                'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+            },
+            'playlist_count': 10,
         },
-        'playlist_count': 10,
-    }]
+        {
+            'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+            'only_matching': True,
+        }
+    ]
     IE_NAME = 'zingmp3:album'
     IE_DESC = 'mp3.zing.vn albums'
 

From 43abd79950b7ba0c3b6e8a26b624e439036b1c7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:05:01 +0600
Subject: [PATCH 1533/2145] [zingmp3:album] Style

---
 youtube_dl/extractor/zingmp3.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 42ac124f0..1059dd75f 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -86,21 +86,18 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
 
 class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
     _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [
-        {
-            'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-            'info_dict': {
-                '_type': 'playlist',
-                'id': 'ZWZBWDAF',
-                'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
-            },
-            'playlist_count': 10,
+    _TESTS = [{
+        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ZWZBWDAF',
+            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
         },
-        {
-            'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
-            'only_matching': True,
-        }
-    ]
+        'playlist_count': 10,
+    }, {
+        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+        'only_matching': True,
+    }]
     IE_NAME = 'zingmp3:album'
     IE_DESC = 'mp3.zing.vn albums'
 

From 8119597d6ff52a671fa359378dfbdee13f24f882 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:08:38 +0600
Subject: [PATCH 1534/2145] [zingmp3] Add fatal flag

---
 youtube_dl/extractor/zingmp3.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 1059dd75f..3f46f9049 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -9,9 +9,11 @@ from ..utils import ExtractorError
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    def _extract_item(self, item):
+    def _extract_item(self, item, fatal=True):
         error_message = item.find('./errormessage').text
         if error_message:
+            if not fatal:
+                return
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, error_message),
                 expected=True)

From f790c43f6e33782c6a6eaaff68bbd30aad875efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:09:21 +0600
Subject: [PATCH 1535/2145] [zingmp3:album] Skip broken items

---
 youtube_dl/extractor/zingmp3.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 3f46f9049..437eecb67 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -45,7 +45,9 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             entries = []
 
             for i, item in enumerate(items, 1):
-                entry = self._extract_item(item)
+                entry = self._extract_item(item, fatal=False)
+                if not entry:
+                    continue
                 entry['id'] = '%s-%d' % (id, i)
                 entries.append(entry)
 

From e6174ee9753e044bdeac22c653f5762376cadbac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:01:20 +0600
Subject: [PATCH 1536/2145] [bbc] Extract legacy playlist embedded media

---
 youtube_dl/extractor/bbc.py | 48 ++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 911ccd757..7bbbc0b90 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -29,6 +29,14 @@ class BBCCoUkIE(InfoExtractor):
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
 
+    _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
+    _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'
+
+    _NAMESPACES = (
+        _MEDIASELECTION_NS,
+        _EMP_PLAYLIST_NS,
+    )
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -194,6 +202,7 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_connection(self, connection, programme_id):
         formats = []
+        kind = connection.get('kind')
         protocol = connection.get('protocol')
         supplier = connection.get('supplier')
         if protocol == 'http':
@@ -219,7 +228,7 @@ class BBCCoUkIE(InfoExtractor):
             else:
                 formats.append({
                     'url': href,
-                    'format_id': supplier,
+                    'format_id': supplier or kind or protocol,
                 })
         elif protocol == 'rtmp':
             application = connection.get('application', 'ondemand')
@@ -239,16 +248,24 @@ class BBCCoUkIE(InfoExtractor):
         return formats
 
     def _extract_items(self, playlist):
-        return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
+        return playlist.findall('./{%s}item' % self._EMP_PLAYLIST_NS)
+
+    def _findall_ns(self, element, xpath):
+        elements = []
+        for ns in self._NAMESPACES:
+            elements.extend(element.findall(xpath % ns))
+        return elements
 
     def _extract_medias(self, media_selection):
-        error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
+        error = media_selection.find('./{%s}error' % self._MEDIASELECTION_NS)
+        if error is None:
+            media_selection.find('./{%s}error' % self._EMP_PLAYLIST_NS)
         if error is not None:
             raise BBCCoUkIE.MediaSelectionError(error.get('id'))
-        return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
+        return self._findall_ns(media_selection, './{%s}media')
 
     def _extract_connections(self, media):
-        return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')
+        return self._findall_ns(media, './{%s}connection')
 
     def _extract_video(self, media, programme_id):
         formats = []
@@ -262,13 +279,14 @@ class BBCCoUkIE(InfoExtractor):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
                 format.update({
-                    'format_id': '%s_%s' % (service, format['format_id']),
                     'width': width,
                     'height': height,
                     'vbr': vbr,
                     'vcodec': vcodec,
                     'filesize': file_size,
                 })
+                if service:
+                    format['format_id'] = '%s_%s' % (service, format['format_id'])
             formats.extend(conn_formats)
         return formats
 
@@ -383,7 +401,7 @@ class BBCCoUkIE(InfoExtractor):
             url, playlist_id, 'Downloading legacy playlist XML')
 
     def _extract_from_legacy_playlist(self, playlist, playlist_id):
-        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+        no_items = playlist.find('./{%s}noItems' % self._EMP_PLAYLIST_NS)
         if no_items is not None:
             reason = no_items.get('reason')
             if reason == 'preAvailability':
@@ -400,8 +418,8 @@ class BBCCoUkIE(InfoExtractor):
             kind = item.get('kind')
             if kind != 'programme' and kind != 'radioProgramme':
                 continue
-            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description_el = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary')
+            title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
+            description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
             description = description_el.text if description_el else None
 
             def get_programme_id(item):
@@ -411,16 +429,18 @@ class BBCCoUkIE(InfoExtractor):
                         if value and re.match(r'^[pb][\da-z]{7}$', value):
                             return value
                 get_from_attributes(item)
-                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                mediator = item.find('./{%s}mediator' % self._EMP_PLAYLIST_NS)
                 if mediator is not None:
                     return get_from_attributes(mediator)
 
             programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
-            # TODO: programme_id can be None and media items can be incorporated right inside
-            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
-            # as f4m and m3u8
-            formats, subtitles = self._download_media_selector(programme_id)
+
+            if programme_id:
+                formats, subtitles = self._download_media_selector(programme_id)
+            else:
+                formats, subtitles = self._process_media_selector(item, playlist_id)
+                programme_id = playlist_id
 
         return programme_id, title, description, duration, formats, subtitles
 

From c936d8cc7b247f79dce2f5749dea5c18a08858fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:14:25 +0600
Subject: [PATCH 1537/2145] [bbc] Add another test

---
 youtube_dl/extractor/bbc.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7bbbc0b90..0a08a57bd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -559,6 +559,19 @@ class BBCIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video embedded with playlist.sxml in data-playable
+        'url': 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani',
+        'info_dict': {
+            'id': '151010_vid_ankara_patlama_ani',
+            'ext': 'mp4',
+            'title': "Ankara'da patlama anı",
+            'timestamp': 1444480325,
+            'upload_date': '20151010',
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         # single video embedded with mediaAssetPage.init() (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',

From b1ec70e4a9fc53b0ec583f48a5262c9f864db40b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:20:30 +0600
Subject: [PATCH 1538/2145] [bbc] Improve data-playable regex

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 0a08a57bd..972abd0d4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -739,7 +739,8 @@ class BBCIE(BBCCoUkIE):
             # http://www.bbc.com/news/world-us-canada-34473351)
             data_playable = self._parse_json(
                 unescapeHTML(self._search_regex(
-                    r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
+                    r'data-playable=(["\'])(?P<json>{.+?})\1', webpage,
+                    'data playable', default='{}', group='json')),
                 programme_id, fatal=False)
             if data_playable:
                 # data-playable has video vpid in settings.playlistObject.items (e.g.

From 78f9d843186977c614c5a0f6004732f5d410cd0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:40:20 +0600
Subject: [PATCH 1539/2145] [bbc] Support playlists of data-playable

---
 youtube_dl/extractor/bbc.py | 73 ++++++++++++++++++++++---------------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 972abd0d4..a15e67114 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -716,6 +716,8 @@ class BBCIE(BBCCoUkIE):
              r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
             webpage, 'date', default=None))
 
+        entries = []
+
         # article with multiple videos embedded with playlist.sxml (e.g.
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
@@ -723,6 +725,48 @@ class BBCIE(BBCCoUkIE):
             entries = [
                 self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
                 for playlist_url in playlists]
+
+        # news article with multiple videos embedded with data-playable
+        data_playables = re.findall(r'data-playable=(["\'])({.+?})\1', webpage)
+        if data_playables:
+            for _, data_playable_json in data_playables:
+                data_playable = self._parse_json(
+                    unescapeHTML(data_playable_json), playlist_id, fatal=False)
+                if not data_playable:
+                    continue
+                settings = data_playable.get('settings', {})
+                if settings:
+                    # data-playable with video vpid in settings.playlistObject.items (e.g.
+                    # http://www.bbc.com/news/world-us-canada-34473351)
+                    playlist_object = settings.get('playlistObject', {})
+                    if playlist_object:
+                        items = playlist_object.get('items')
+                        if items and isinstance(items, list):
+                            title = playlist_object['title']
+                            description = playlist_object.get('summary')
+                            duration = int_or_none(items[0].get('duration'))
+                            programme_id = items[0].get('vpid')
+                            formats, subtitles = self._download_media_selector(programme_id)
+                            self._sort_formats(formats)
+                            entries.append({
+                                'id': programme_id,
+                                'title': title,
+                                'description': description,
+                                'timestamp': timestamp,
+                                'duration': duration,
+                                'formats': formats,
+                                'subtitles': subtitles,
+                            })
+                    else:
+                        # data-playable without vpid but with a playlist.sxml URLs
+                        # in otherSettings.playlist (e.g.
+                        # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
+                        playlist = data_playable.get('otherSettings', {}).get('playlist', {})
+                        if playlist:
+                            entries.append(self._extract_from_playlist_sxml(
+                                playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
+
+        if entries:
             playlist_title = self._og_search_title(webpage)
             playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
@@ -733,35 +777,6 @@ class BBCIE(BBCCoUkIE):
              r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
             webpage, 'vpid', default=None)
 
-        duration = None
-        if not programme_id:
-            # single video in news article embedded with data-playable (e.g.
-            # http://www.bbc.com/news/world-us-canada-34473351)
-            data_playable = self._parse_json(
-                unescapeHTML(self._search_regex(
-                    r'data-playable=(["\'])(?P<json>{.+?})\1', webpage,
-                    'data playable', default='{}', group='json')),
-                programme_id, fatal=False)
-            if data_playable:
-                # data-playable has video vpid in settings.playlistObject.items (e.g.
-                # http://www.bbc.com/news/world-us-canada-34473351)
-                settings = data_playable.get('settings', {})
-                if settings:
-                    playlist_object = settings.get('playlistObject', {})
-                    if playlist_object:
-                        items = playlist_object.get('items')
-                        if items and isinstance(items, list):
-                            duration = int_or_none(items[0].get('duration'))
-                            programme_id = items[0].get('vpid')
-                if not programme_id:
-                    # data-playable has no vpid but has a playlist.sxml URLs
-                    # in otherSettings.playlist (e.g.
-                    # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
-                    playlist = data_playable.get('otherSettings', {}).get('playlist', {})
-                    if playlist:
-                        return self._extract_from_playlist_sxml(
-                            playlist.get('progressiveDownloadUrl'), playlist_id, timestamp)
-
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             self._sort_formats(formats)

From f6295bcb04c0e518369c31a44b691ad276fcc22f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:45:15 +0600
Subject: [PATCH 1540/2145] [bbc] Remove duration

---
 youtube_dl/extractor/bbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a15e67114..fad1f9ca7 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -794,7 +794,6 @@ class BBCIE(BBCCoUkIE):
                 'title': title,
                 'description': description,
                 'timestamp': timestamp,
-                'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
             }

From 6a747190605229e9cfba5450cf0ecaf435b7a85e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:56:55 +0600
Subject: [PATCH 1541/2145] [bbc] Update tests

---
 youtube_dl/extractor/bbc.py | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index fad1f9ca7..4eae4f52e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -501,8 +501,7 @@ class BBCIE(BBCCoUkIE):
     ]
 
     _TESTS = [{
-        # article with multiple videos embedded with data-media-meta containing
-        # playlist.sxml, externalId and no direct video links
+        # article with multiple videos embedded with data-playable containing vpids
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
             'id': 'world-europe-32668511',
@@ -511,7 +510,7 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 2,
     }, {
-        # article with multiple videos embedded with data-media-meta (more videos)
+        # article with multiple videos embedded with data-playable (more videos)
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
@@ -522,6 +521,7 @@ class BBCIE(BBCCoUkIE):
         'skip': 'Save time',
     }, {
         # article with multiple videos embedded with `new SMP()`
+        # broken
         'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
         'info_dict': {
             'id': '3662a707-0af9-3149-963f-47bea720b460',
@@ -529,7 +529,7 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 18,
     }, {
-        # single video embedded with mediaAssetPage.init()
+        # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
@@ -544,9 +544,9 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # article with single video embedded with data-media-meta containing
-        # direct video links (for now these are extracted) and playlist.xml (with
-        # media items as f4m and m3u8 - currently unsupported)
+        # article with single video embedded with data-playable containing XML playlist
+        # with direct video links as progressiveDownloadUrl (for now these are extracted)
+        # and playlist with f4m and m3u8 as streamingUrl
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
         'info_dict': {
             'id': '150615_telabyad_kentin_cogu',
@@ -560,20 +560,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # single video embedded with playlist.sxml in data-playable
-        'url': 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani',
-        'info_dict': {
-            'id': '151010_vid_ankara_patlama_ani',
-            'ext': 'mp4',
-            'title': "Ankara'da patlama anı",
-            'timestamp': 1444480325,
-            'upload_date': '20151010',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        # single video embedded with mediaAssetPage.init() (regional section)
+        # single video embedded with data-playable containing XML playlists (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
         'info_dict': {
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
@@ -629,7 +616,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # single video with playlist.sxml URL
+        # single video with playlist.sxml URL in playlist param
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
@@ -643,7 +630,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # article with multiple videos embedded with playlist.sxml
+        # article with multiple videos embedded with playlist.sxml in playlist param
         'url': 'http://www.bbc.com/sport/0/football/34475836',
         'info_dict': {
             'id': '34475836',

From ae8bdfd1a1548c83ab7df378096da927b5374a29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:25:09 +0600
Subject: [PATCH 1542/2145] [bbc] Extract article JSON and actualize tests

---
 youtube_dl/extractor/bbc.py | 45 ++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4eae4f52e..b98db95b9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    remove_end,
     unescapeHTML,
 )
 from ..compat import compat_HTTPError
@@ -533,7 +534,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
             'timestamp': 1427219242,
@@ -552,7 +553,6 @@ class BBCIE(BBCCoUkIE):
             'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
             'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
-            'duration': 47,
             'timestamp': 1434397334,
             'upload_date': '20150615',
         },
@@ -566,7 +566,6 @@ class BBCIE(BBCCoUkIE):
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
             'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
-            'duration': 87,
             'timestamp': 1434713142,
             'upload_date': '20150619',
         },
@@ -578,7 +577,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
         'info_dict': {
             'id': 'p02w6qjc',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
         },
@@ -605,11 +604,11 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
-            'timestamp': 1368473503,
-            'upload_date': '20130513',
+            'timestamp': 1415867444,
+            'upload_date': '20141113',
         },
         'params': {
             # rtmp download
@@ -620,9 +619,8 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
-            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
             'duration': 140,
         },
         'params': {
@@ -697,11 +695,26 @@ class BBCIE(BBCCoUkIE):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        timestamp = parse_iso8601(self._search_regex(
-            [r'"datePublished":\s*"([^"]+)',
-             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
-             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
-            webpage, 'date', default=None))
+        timestamp = None
+        playlist_title = None
+        playlist_description = None
+
+        ld = self._parse_json(
+            self._search_regex(
+                r'(?s)<script type="application/ld\+json">(.+?)</script>',
+                webpage, 'ld json', default='{}'),
+            playlist_id, fatal=False)
+        if ld:
+            timestamp = parse_iso8601(ld.get('datePublished'))
+            playlist_title = ld.get('headline')
+            playlist_description = ld.get('articleBody')
+
+        if not timestamp:
+            timestamp = parse_iso8601(self._search_regex(
+                [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+                 r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
+                 r'"datePublished":\s*"([^"]+)',],
+                webpage, 'date', default=None))
 
         entries = []
 
@@ -754,8 +767,8 @@ class BBCIE(BBCCoUkIE):
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
         if entries:
-            playlist_title = self._og_search_title(webpage)
-            playlist_description = self._og_search_description(webpage, default=None)
+            playlist_title = playlist_title or remove_end(self._og_search_title(webpage), ' - BBC News')
+            playlist_description = playlist_description or self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)

From 55ebae26f937a6a3a1fcb78d2f797972080f8b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:37:39 +0600
Subject: [PATCH 1543/2145] [bbc] Prefer iptv-all mediaset

---
 youtube_dl/extractor/bbc.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b98db95b9..f3ded3f9c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -493,6 +493,9 @@ class BBCIE(BBCCoUkIE):
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
     _MEDIASELECTOR_URLS = [
+        # Provides HQ HLS streams but fails with geolocation in some cases when it's
+        # even not geo restricted at all
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         # Provides more formats, namely direct mp4 links, but fails on some videos with
         # notukerror for non UK (?) users (e.g.
         # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
@@ -534,8 +537,9 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'md5:2868290467291b37feda7863f7a83f54',
             'duration': 47,
             'timestamp': 1427219242,
             'upload_date': '20150324',
@@ -577,7 +581,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
         'info_dict': {
             'id': 'p02w6qjc',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
         },
@@ -604,7 +608,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
             'timestamp': 1415867444,
@@ -619,7 +623,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
             'duration': 140,
         },

From 6f7893653c86c620099d7bf0e3bd4951be8b4ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:39:28 +0600
Subject: [PATCH 1544/2145] [bbc] PEP 8

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f3ded3f9c..68995f81e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -717,7 +717,7 @@ class BBCIE(BBCCoUkIE):
             timestamp = parse_iso8601(self._search_regex(
                 [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
                  r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
-                 r'"datePublished":\s*"([^"]+)',],
+                 r'"datePublished":\s*"([^"]+)'],
                 webpage, 'date', default=None))
 
         entries = []

From f4076bb73686484310a3a2b7c0c4c20db6e186bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 19:56:55 +0100
Subject: [PATCH 1545/2145] [vimeo] extract m3u8 manifest and bitrate

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7dd52627d..93638d6b2 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -397,8 +397,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'format_id': format_id,
                     'width': file_info.get('width'),
                     'height': file_info.get('height'),
+                    'tbr': file_info.get('bitrate'),
                 })
         formats = []
+        hls = config_files.get("hls")
+        if hls:
+            formats = self._extract_m3u8_formats(hls['all'], video_id, m3u8_id='hls')
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:

From 58cd7e173e70ae40a79bb10e08b2c2ea02bc8248 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 12:28:12 +0100
Subject: [PATCH 1546/2145] [adultswim] detect when video needs authentication

---
 youtube_dl/extractor/adultswim.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 27de07587..0eb21b16d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -41,7 +41,8 @@ class AdultSwimIE(InfoExtractor):
             'id': 'rQxZvXQ4ROaSOqq-or2Mow',
             'title': 'Rick and Morty - Pilot',
             'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-        }
+        },
+        'skip': 'This video is only available for registered users',
     }, {
         'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
         'playlist': [
@@ -84,7 +85,10 @@ class AdultSwimIE(InfoExtractor):
     def find_video_info(collection, slug):
         for video in collection.get('videos'):
             if video.get('slug') == slug:
-                return video
+                if video.get('auth'):
+                    raise ExtractorError('This video is only available for registered users', expected=True)
+                else:
+                    return video
 
     @staticmethod
     def find_collection_by_linkURL(collections, linkURL):
@@ -97,7 +101,10 @@ class AdultSwimIE(InfoExtractor):
         for collection in collections:
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
-                    return collection, video
+                    if video.get('auth'):
+                        raise ExtractorError('This video is only available for registered users', expected=True)
+                    else:
+                        return collection, video
         return None, None
 
     def _real_extract(self, url):
@@ -128,6 +135,8 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
+                    if video_info.get('auth'):
+                        raise ExtractorError('This video is only available for registered users', expected=True)
                 else:
                     raise ExtractorError('Unable to find video info')
 

From 00cde0b8dc3a534a929b88da086cc540dc212ea4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 17:57:05 +0100
Subject: [PATCH 1547/2145] [adultswim] raise ExtractorError if no clips in
 video_info

---
 youtube_dl/extractor/adultswim.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 0eb21b16d..35606878d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -85,10 +85,7 @@ class AdultSwimIE(InfoExtractor):
     def find_video_info(collection, slug):
         for video in collection.get('videos'):
             if video.get('slug') == slug:
-                if video.get('auth'):
-                    raise ExtractorError('This video is only available for registered users', expected=True)
-                else:
-                    return video
+                return video
 
     @staticmethod
     def find_collection_by_linkURL(collections, linkURL):
@@ -101,10 +98,7 @@ class AdultSwimIE(InfoExtractor):
         for collection in collections:
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
-                    if video.get('auth'):
-                        raise ExtractorError('This video is only available for registered users', expected=True)
-                    else:
-                        return collection, video
+                    return collection, video
         return None, None
 
     def _real_extract(self, url):
@@ -135,15 +129,18 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
-                    if video_info.get('auth'):
-                        raise ExtractorError('This video is only available for registered users', expected=True)
                 else:
                     raise ExtractorError('Unable to find video info')
 
             show = bootstrapped_data['show']
             show_title = show['title']
             stream = video_info.get('stream')
-            clips = [stream] if stream else video_info['clips']
+            clips = [stream] if stream else video_info.get('clips')
+            if not clips:
+                if video_info.get('auth'):
+                    raise ExtractorError('This video is only available for registered users', expected=True)
+                else:
+                    raise ExtractorError('Unable to find clips')
             segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']

From 75f105d455bd0a2226ce4fa78b56c7a344ad6bd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 01:04:41 +0600
Subject: [PATCH 1548/2145] [adultswim] Clarify no media message

---
 youtube_dl/extractor/adultswim.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 35606878d..130afe791 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -137,10 +137,11 @@ class AdultSwimIE(InfoExtractor):
             stream = video_info.get('stream')
             clips = [stream] if stream else video_info.get('clips')
             if not clips:
-                if video_info.get('auth'):
-                    raise ExtractorError('This video is only available for registered users', expected=True)
-                else:
-                    raise ExtractorError('Unable to find clips')
+                raise ExtractorError(
+                    'This video is only available via cable service provider subscription that'
+                    ' is not currently supported. You may want to use --cookies.'
+                    if video_info.get('auth') is True else 'Unable to find stream or clips',
+                    expected=True)
             segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']

From e5c209a1bcea206bee684914599c84acf886487c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 20:34:10 +0100
Subject: [PATCH 1549/2145] [vimeo] add parameters to _extract_m3u8_formats and
 sort formats

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 93638d6b2..2ea5f0b79 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -402,11 +402,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
         hls = config_files.get("hls")
         if hls:
-            formats = self._extract_m3u8_formats(hls['all'], video_id, m3u8_id='hls')
+            formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:
             raise ExtractorError('No known codec found')
+        self._sort_formats(formats)
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From fff496c689e44ac96909cf55c9ae746fb6b14e07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 20:45:34 +0100
Subject: [PATCH 1550/2145] [vimeo] remove check for empty formats

---
 youtube_dl/extractor/vimeo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2ea5f0b79..2051ac9de 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -405,8 +405,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
             formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
-        if len(formats) == 0:
-            raise ExtractorError('No known codec found')
         self._sort_formats(formats)
 
         subtitles = {}

From 35a3ff1d337edd527c73db133d87ed23ca4469f5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 21:05:29 +0100
Subject: [PATCH 1551/2145] [vimeo] always convert width, height and bitrate to
 int

---
 youtube_dl/extractor/vimeo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2051ac9de..97590d220 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -395,14 +395,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'ext': codec_extension,
                     'url': video_url,
                     'format_id': format_id,
-                    'width': file_info.get('width'),
-                    'height': file_info.get('height'),
-                    'tbr': file_info.get('bitrate'),
+                    'width': int_or_none(file_info.get('width')),
+                    'height': int_or_none(file_info.get('height')),
+                    'tbr': int_or_none(file_info.get('bitrate')),
                 })
         formats = []
-        hls = config_files.get("hls")
-        if hls:
-            formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+        m3u8_url = config_files.get('hls', {}).get('all')
+        if m3u8_url:
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From 68f3b61f0e278891b5a0d6557d297dcfd3fb53cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 02:22:42 +0600
Subject: [PATCH 1552/2145] [vimeo] Update tests

---
 youtube_dl/extractor/vimeo.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 97590d220..9c173ce07 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -84,12 +84,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'info_dict': {
                 'id': '56015672',
                 'ext': 'mp4',
-                "upload_date": "20121220",
-                "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
-                "uploader_id": "user7108434",
-                "uploader": "Filippo Valsorda",
-                "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
-                "duration": 10,
+                'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
+                'description': 'md5:2d3305bad981a06ff79f027f19865021',
+                'upload_date': '20121220',
+                'uploader_id': 'user7108434',
+                'uploader': 'Filippo Valsorda',
+                'duration': 10,
             },
         },
         {
@@ -102,7 +102,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'openstreetmapus',
                 'uploader': 'OpenStreetMap US',
                 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
-                'description': 'md5:380943ec71b89736ff4bf27183233d09',
+                'description': 'md5:fd69a7b8d8c34a4e1d2ec2e4afd6ec30',
                 'duration': 1595,
             },
         },
@@ -156,7 +156,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
         {
             'url': 'http://vimeo.com/76979871',
-            'md5': '3363dd6ffebe3784d56f4132317fd446',
             'note': 'Video with subtitles',
             'info_dict': {
                 'id': '76979871',

From 7c84562945cc5c37979b0d8d786118b1fe31f731 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 02:31:37 +0600
Subject: [PATCH 1553/2145] [vimeo] Fix password protected videos

---
 youtube_dl/extractor/vimeo.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9c173ce07..346eb60b4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -39,10 +39,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             return
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
-        token = self._extract_xsrft(webpage)
-        vuid = self._search_regex(
-            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
-            webpage, 'vuid', group='vuid')
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         data = urlencode_postdata({
             'action': 'login',
             'email': username,
@@ -56,10 +53,14 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
-    def _extract_xsrft(self, webpage):
-        return self._search_regex(
+    def _extract_xsrft_and_vuid(self, webpage):
+        xsrft = self._search_regex(
             r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
             webpage, 'login token', group='xsrft')
+        vuid = self._search_regex(
+            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
+            webpage, 'vuid', group='vuid')
+        return xsrft, vuid
 
 
 class VimeoIE(VimeoBaseInfoExtractor):
@@ -201,7 +202,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._extract_xsrft(webpage)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -211,6 +212,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'clip_v=1; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,
@@ -465,7 +467,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
-        token = self._extract_xsrft(webpage)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)
@@ -474,6 +476,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'vuid=%s' % vuid)
         self._set_cookie('vimeo.com', 'xsrft', token)
 
         return self._download_webpage(

From 70cb4d51c91c5b6c72b8564258cb161bcc68626c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 00:45:23 +0100
Subject: [PATCH 1554/2145] [bild] extract info from json request

---
 youtube_dl/extractor/bild.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 4d8cce1ef..ea84f20f2 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    fix_xml_ampersands,
+    unescapeHTML,
 )
 
 
@@ -17,7 +17,7 @@ class BildIE(InfoExtractor):
         'info_dict': {
             'id': '38184146',
             'ext': 'mp4',
-            'title': 'BILD hat sie getestet',
+            'title': 'Das können die  neuen iPads ',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
@@ -27,16 +27,13 @@ class BildIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
-        doc = self._download_xml(xml_url, video_id, transform_source=fix_xml_ampersands)
-
-        duration = int_or_none(doc.attrib.get('duration'), scale=1000)
+        video_data = self._download_json(url.split(".bild.html")[0] + ",view=json.bild.html", video_id)
 
         return {
             'id': video_id,
-            'title': doc.attrib['ueberschrift'],
-            'description': doc.attrib.get('text'),
-            'url': doc.attrib['src'],
-            'thumbnail': doc.attrib.get('img'),
-            'duration': duration,
+            'title': unescapeHTML(video_data['title']),
+            'description': unescapeHTML(video_data.get('description')),
+            'url': video_data['clipList'][0]['srces'][0]['src'],
+            'thumbnail': video_data.get('poster'),
+            'duration': int_or_none(video_data.get('durationSec')),
         }

From d8348c351d5ed4bb820b190e42b9a02e2dfc27a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:16:51 +0600
Subject: [PATCH 1555/2145] [bild] Strip title

---
 youtube_dl/extractor/bild.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index ea84f20f2..1a0184861 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -17,21 +17,22 @@ class BildIE(InfoExtractor):
         'info_dict': {
             'id': '38184146',
             'ext': 'mp4',
-            'title': 'Das können die  neuen iPads ',
+            'title': 'Das können die  neuen iPads',
+            'description': 'md5:a4058c4fa2a804ab59c00d7244bbf62f',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
-            'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_data = self._download_json(url.split(".bild.html")[0] + ",view=json.bild.html", video_id)
+        video_data = self._download_json(
+            url.split('.bild.html')[0] + ',view=json.bild.html', video_id)
 
         return {
             'id': video_id,
-            'title': unescapeHTML(video_data['title']),
+            'title': unescapeHTML(video_data['title']).strip(),
             'description': unescapeHTML(video_data.get('description')),
             'url': video_data['clipList'][0]['srces'][0]['src'],
             'thumbnail': video_data.get('poster'),

From 03e3b4e1198631b30914e8669b01bf1825a4385c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 23:36:31 +0100
Subject: [PATCH 1556/2145] [expotv] parse m3u8 manifest

---
 youtube_dl/extractor/expotv.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index a38b773e8..23a38c7c1 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -33,20 +33,24 @@ class ExpoTVIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         player_key = self._search_regex(
             r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
-        config_url = 'http://client.expotv.com/video/config/%s/%s' % (
-            video_id, player_key)
         config = self._download_json(
-            config_url, video_id,
-            note='Downloading video configuration')
+                'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
+                video_id,
+                note='Downloading video configuration')
 
-        formats = [{
-            'url': fcfg['file'],
-            'height': int_or_none(fcfg.get('height')),
-            'format_note': fcfg.get('label'),
-            'ext': self._search_regex(
-                r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
-                'file extension', default=None),
-        } for fcfg in config['sources']]
+        formats = []
+        for fcfg in config['sources']:
+            if fcfg['type'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(fcfg['file'], video_id))
+            else:
+                formats.append({
+                    'url': fcfg['file'],
+                    'height': int_or_none(fcfg.get('height')),
+                    'format_id': fcfg.get('label'),
+                    'ext': self._search_regex(
+                        r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
+                        'file extension', default=None),
+                })
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 7d49502ab0990972bdb479e6bf32dbc8ffdb3e1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:28:00 +0600
Subject: [PATCH 1557/2145] [bild] Make more robust and improve hls extraction

---
 youtube_dl/extractor/expotv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index 23a38c7c1..1585a03bb 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -34,22 +34,25 @@ class ExpoTVIE(InfoExtractor):
         player_key = self._search_regex(
             r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
         config = self._download_json(
-                'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
-                video_id,
-                note='Downloading video configuration')
+            'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
+            video_id, 'Downloading video configuration')
 
         formats = []
         for fcfg in config['sources']:
-            if fcfg['type'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(fcfg['file'], video_id))
+            media_url = fcfg.get('file')
+            if not media_url:
+                continue
+            if fcfg.get('type') == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
             else:
                 formats.append({
-                    'url': fcfg['file'],
+                    'url': media_url,
                     'height': int_or_none(fcfg.get('height')),
                     'format_id': fcfg.get('label'),
                     'ext': self._search_regex(
-                        r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
-                        'file extension', default=None),
+                        r'filename=.*\.([a-z0-9_A-Z]+)&', media_url,
+                        'file extension', default=None) or fcfg.get('type'),
                 })
         self._sort_formats(formats)
 

From da4daed5ef77b4a7219b1786978065a3606b85bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:36:23 +0600
Subject: [PATCH 1558/2145] [vimeo] Do not fail when no hls formats

---
 youtube_dl/extractor/vimeo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 346eb60b4..6ee3069a8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -403,7 +403,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
         m3u8_url = config_files.get('hls', {}).get('all')
         if m3u8_url:
-            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+            if m3u8_formats:
+                formats.append(m3u8_formats)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From 1bd390358205728823ff38a51b12ae35f9468929 Mon Sep 17 00:00:00 2001
From: PC <tioocnt@yandex.com>
Date: Tue, 6 Oct 2015 22:28:58 +0100
Subject: [PATCH 1559/2145] chaturbate streams

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/chaturbate.py | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 youtube_dl/extractor/chaturbate.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ace1cc2c..75720843c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -76,6 +76,7 @@ from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
+from .chaturbate import ChaturbateIE
 from .chilloutzone import ChilloutzoneIE
 from .chirbit import (
     ChirbitIE,
diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
new file mode 100644
index 000000000..5e24e1e4f
--- /dev/null
+++ b/youtube_dl/extractor/chaturbate.py
@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+from .common import InfoExtractor
+
+
+class ChaturbateIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chaturbate\.com/(?P<id>[^/]+)/?$'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        m3u8_url = self._search_regex(r"'(https?://.*?\.m3u8)'", webpage, 'playlist')
+
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': self._live_title(video_id),
+            'description': self._html_search_meta('description', webpage, 'description'),
+            'is_live': True,
+            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % (video_id,),
+            'formats': formats,
+        }

From 0f61db4469db4c17712b8a3f9e527e845ec96ed4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 20:35:22 +0600
Subject: [PATCH 1560/2145] [chaturbate] Improve and capture error message

---
 youtube_dl/extractor/chaturbate.py | 36 +++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 5e24e1e4f..0b67ba67d 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -1,24 +1,50 @@
-# encoding: utf-8
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class ChaturbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chaturbate\.com/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.chaturbate.com/siswet19/',
+        'info_dict': {
+            'id': 'siswet19',
+            'ext': 'mp4',
+            'title': 're:^siswet19 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'age_limit': 18,
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://en.chaturbate.com/siswet19/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_url = self._search_regex(r"'(https?://.*?\.m3u8)'", webpage, 'playlist')
+        m3u8_url = self._search_regex(
+            r'src=(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage,
+            'playlist', default=None, group='url')
+
+        if not m3u8_url:
+            error = self._search_regex(
+                r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
+                webpage, 'error', group='error')
+            raise ExtractorError(error, expected=True)
 
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video_id,
             'title': self._live_title(video_id),
-            'description': self._html_search_meta('description', webpage, 'description'),
+            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
+            'age_limit': self._rta_search(webpage),
             'is_live': True,
-            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % (video_id,),
             'formats': formats,
         }

From f101079ae02560fee2df1ea0cacb1989eefc8e6c Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Sun, 11 Oct 2015 19:17:24 +0200
Subject: [PATCH 1561/2145] [downloader/rtmp] Respect --no-continue option

now when downloading a "rtmp-file" the --no-continue option works!
---
 youtube_dl/downloader/rtmp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 7d19bb808..f1d219ba9 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
-        continue_dl = info_dict.get('continuedl', True)
+        continue_dl = self.params.get('continuedl', True)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

From 964e7b2dd09099bd021a770522684716f341db43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Oct 2015 00:43:54 +0600
Subject: [PATCH 1562/2145] [downloader/common] Always skip "already
 downloaded" check when outputting to stdout

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 97e755d4b..29a4500d3 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -325,7 +325,7 @@ class FileDownloader(object):
         )
 
         # Check file already present
-        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
+        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 57d1db8dd00b833c08b3ac74fbd589a0216775f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 11 Oct 2015 22:45:13 +0200
Subject: [PATCH 1563/2145] [rtbf] Fix extraction (closes #7133)

---
 youtube_dl/extractor/rtbf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e4215d546..04a66df90 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -36,7 +36,7 @@ class RTBFIE(InfoExtractor):
 
         data = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-video="([^"]+)"', webpage, 'data video')),
+                r'data-media="([^"]+)"', webpage, 'data video')),
             video_id)
 
         if data.get('provider').lower() == 'youtube':

From cd7364a89c69c4c19e4b9d4e54d08fd1eff9c51d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 12 Oct 2015 06:37:20 +0200
Subject: [PATCH 1564/2145] release 2015.10.12

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5beb2ecd4..dc0354095 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,6 +81,7 @@
  - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
+ - **Chaturbate**
  - **Chilloutzone**
  - **chirbit**
  - **chirbit:profile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index faae8a2d7..0908e963d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.09'
+__version__ = '2015.10.12'

From 73e732eb6b5fb2edd61701ae110cc52fe6836364 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 12 Oct 2015 16:34:55 +0800
Subject: [PATCH 1565/2145] [vimeo] Fix m3u8 formats

Reported at
https://github.com/rg3/youtube-dl/pull/7126#issuecomment-147327584
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6ee3069a8..027f47ee3 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -406,7 +406,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             m3u8_formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
             if m3u8_formats:
-                formats.append(m3u8_formats)
+                formats.extend(m3u8_formats)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From ee2edd838a1e8770488e695c380943ded44d0983 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 13 Oct 2015 00:53:05 +0200
Subject: [PATCH 1566/2145] release 2015.10.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0908e963d..aaa43d315 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.12'
+__version__ = '2015.10.13'

From 5946cda7c6f2e4a7eb90fff6f10c66af0ff2a0d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Oct 2015 21:04:39 +0600
Subject: [PATCH 1567/2145] [beeg] Fix extraction (Closes #7155)

---
 youtube_dl/extractor/beeg.py | 68 +++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index b38057f2f..e6c928699 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,65 +1,67 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '1bff67111adb785c51d1b42959ec10e5',
+        'md5': '46c384def73b33dbc581262e5ee67cef',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',
             'title': 'Sultry Striptease',
-            'description': 'md5:6db3c6177972822aaba18652ff59c773',
-            'categories': list,  # NSFW
-            'thumbnail': 're:https?://.*\.jpg$',
+            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
+            'timestamp': 1391813355,
+            'upload_date': '20140207',
+            'duration': 383,
+            'tags': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        quality_arr = self._search_regex(
-            r'(?s)var\s+qualityArr\s*=\s*{\s*(.+?)\s*}', webpage, 'quality formats')
-
-        formats = [{
-            'url': fmt[1],
-            'format_id': fmt[0],
-            'height': int(fmt[0][:-1]),
-        } for fmt in re.findall(r"'([^']+)'\s*:\s*'([^']+)'", quality_arr)]
+        video = self._download_json(
+            'http://beeg.com/api/v1/video/%s' % video_id, video_id)
 
+        formats = []
+        for format_id, video_url in video.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            if not height:
+                continue
+            formats.append({
+                'url': self._proto_relative_url(video_url.replace('{DATA_MARKERS}', ''), 'http:'),
+                'format_id': format_id,
+                'height': int(height),
+            })
         self._sort_formats(formats)
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*beeg\.?</title>', webpage, 'title')
+        title = video['title']
+        video_id = video.get('id') or video_id
+        display_id = video.get('code')
+        description = video.get('desc')
 
-        description = self._html_search_regex(
-            r'<meta name="description" content="([^"]*)"',
-            webpage, 'description', fatal=False)
-        thumbnail = self._html_search_regex(
-            r'\'previewer.url\'\s*:\s*"([^"]*)"',
-            webpage, 'thumbnail', fatal=False)
+        timestamp = parse_iso8601(video.get('date'), ' ')
+        duration = int_or_none(video.get('duration'))
 
-        categories_str = self._html_search_regex(
-            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
-            'categories': categories,
+            'timestamp': timestamp,
+            'duration': duration,
+            'tags': tags,
             'formats': formats,
             'age_limit': 18,
         }

From 3eeff489e80838cd2d1b3f55fc96db747386fd50 Mon Sep 17 00:00:00 2001
From: DesweR <egomail@inbox.ru>
Date: Tue, 13 Oct 2015 16:29:16 +0700
Subject: [PATCH 1568/2145] Extract thumbnail url

---
 youtube_dl/extractor/yandexmusic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 4098e4629..6842f834f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -46,6 +46,14 @@ class YandexMusicTrackIE(InfoExtractor):
                 % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
     def _get_track_info(self, track):
+        album = track['albums'][0]
+        a_thumb = None
+
+        if 'coverUri' in album:
+            a_thumb = album['coverUri']
+            if a_thumb:
+                a_thumb = 'http://' + a_thumb.replace('%%', '1000x1000')
+
         return {
             'id': track['id'],
             'ext': 'mp3',
@@ -53,6 +61,7 @@ class YandexMusicTrackIE(InfoExtractor):
             'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
+            'thumbnail': a_thumb,
         }
 
     def _real_extract(self, url):

From b30c4992a93d411f4f89faf2af153fc580138a90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 13 Oct 2015 21:14:33 +0200
Subject: [PATCH 1569/2145] [channel9] Return a single dictionary for single
 videos (closes #7086)

Returning a list is deprecated.
---
 youtube_dl/extractor/channel9.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 3dfc24f5b..79fd0a30e 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -224,12 +224,12 @@ class Channel9IE(InfoExtractor):
         if contents is None:
             return contents
 
-        authors = self._extract_authors(html)
+        if len(contents) > 1:
+            raise ExtractorError('Got more than one entry')
+        result = contents[0]
+        result['authors'] = self._extract_authors(html)
 
-        for content in contents:
-            content['authors'] = authors
-
-        return contents
+        return result
 
     def _extract_session(self, html, content_path):
         contents = self._extract_content(html, content_path)

From 506e261d2073d8c00d5b43d272e8173cb0d63728 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 13 Oct 2015 21:18:30 +0200
Subject: [PATCH 1570/2145] [channel9] strip 'session_day'

---
 youtube_dl/extractor/channel9.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 79fd0a30e..1ce004932 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -158,7 +158,7 @@ class Channel9IE(InfoExtractor):
 
     def _extract_session_day(self, html):
         m = re.search(r'<li class="day">\s*<a href="/Events/[^"]+">(?P<day>[^<]+)</a>\s*</li>', html)
-        return m.group('day') if m is not None else None
+        return m.group('day').strip() if m is not None else None
 
     def _extract_session_room(self, html):
         m = re.search(r'<li class="room">\s*(?P<room>.+?)\s*</li>', html)

From 3dc582e5ea69af4ad7f51d30c1d87cf93aa6b72b Mon Sep 17 00:00:00 2001
From: kaspi <je326@hotmail.com>
Date: Mon, 12 Oct 2015 01:25:57 -0400
Subject: [PATCH 1571/2145] [fczenit] Add extractor

Closes #7143.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fczenit.py  | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/fczenit.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 75720843c..f6d185818 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -167,6 +167,7 @@ from .extremetube import ExtremeTubeIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
+from .fczenit import FczenitIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dl/extractor/fczenit.py
new file mode 100644
index 000000000..f1f150ef2
--- /dev/null
+++ b/youtube_dl/extractor/fczenit.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class FczenitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/gl(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://fc-zenit.ru/video/gl6785/',
+        'md5': '458bacc24549173fe5a5aa29174a5606',
+        'info_dict': {
+            'id': '6785',
+            'ext': 'mp4',
+            'title': '«Зенит-ТВ»: как Олег Шатов играл против «Урала»',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_title = self._html_search_regex(r'<div class=\"photoalbum__title\">([^<]+)', webpage, 'title')
+
+        bitrates_raw = self._html_search_regex(r'bitrates:.*\n(.*)\]', webpage, 'video URL')
+        bitrates = re.findall(r'url:.?\'(.+?)\'.*?bitrate:.?([0-9]{3}?)', bitrates_raw)
+
+        formats = [{
+            'url': furl,
+            'tbr': tbr,
+        } for furl, tbr in bitrates]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_title,
+            'formats': formats,
+        }

From 26669ea3cf596f2ea4bce9e21ce73c1d8fc3ff72 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 16:51:37 +0100
Subject: [PATCH 1572/2145] [5min] extract more video info and formats

Closes #7124.
---
 youtube_dl/extractor/fivemin.py | 84 ++++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 157094e8c..2955965d9 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -2,11 +2,15 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_str,
     compat_urllib_parse,
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    parse_duration,
+    replace_extension,
 )
 
 
@@ -28,6 +32,7 @@ class FiveMinIE(InfoExtractor):
                 'id': '518013791',
                 'ext': 'mp4',
                 'title': 'iPad Mini with Retina Display Review',
+                'duration': 177,
             },
         },
         {
@@ -38,9 +43,52 @@ class FiveMinIE(InfoExtractor):
                 'id': '518086247',
                 'ext': 'mp4',
                 'title': 'How to Make a Next-Level Fruit Salad',
+                'duration': 184,
             },
         },
     ]
+    _ERRORS = {
+        'ErrorVideoNotExist': 'We\'re sorry, but the video you are trying to watch does not exist.',
+        'ErrorVideoNoLongerAvailable': 'We\'re sorry, but the video you are trying to watch is no longer available.',
+        'ErrorVideoRejected': 'We\'re sorry, but the video you are trying to watch has been removed.',
+        'ErrorVideoUserNotGeo': 'We\'re sorry, but the video you are trying to watch cannot be viewed from your current location.',
+        'ErrorVideoLibraryRestriction': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
+        'ErrorExposurePermission': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
+    }
+    _QUALITIES = {
+        1: {
+            'width': 640,
+            'height': 360,
+        },
+        2: {
+            'width': 854,
+            'height': 480,
+        },
+        4: {
+            'width': 1280,
+            'height': 720,
+        },
+        8: {
+            'width': 1920,
+            'height': 1080,
+        },
+        16: {
+            'width': 640,
+            'height': 360,
+        },
+        32: {
+            'width': 854,
+            'height': 480,
+        },
+        64: {
+            'width': 1280,
+            'height': 720,
+        },
+        128: {
+            'width': 640,
+            'height': 360,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -59,26 +107,36 @@ class FiveMinIE(InfoExtractor):
             'https://syn.5min.com/handlers/SenseHandler.ashx?' + query,
             video_id)
         if not response['success']:
-            err_msg = response['errorMessage']
-            if err_msg == 'ErrorVideoUserNotGeo':
-                msg = 'Video not available from your location'
-            else:
-                msg = 'Aol said: %s' % err_msg
-            raise ExtractorError(msg, expected=True, video_id=video_id)
+            raise ExtractorError(
+                '%s said: %s' % (
+                    self.IE_NAME,
+                    self._ERRORS.get(response['errorMessage'], response['errorMessage'])),
+                expected=True)
         info = response['binding'][0]
 
-        second_id = compat_str(int(video_id[:-2]) + 1)
         formats = []
-        for quality, height in [(1, 320), (2, 480), (4, 720), (8, 1080)]:
-            if any(r['ID'] == quality for r in info['Renditions']):
+        parsed_video_url = compat_urllib_parse_urlparse(compat_parse_qs(
+            compat_urllib_parse_urlparse(info['EmbededURL']).query)['videoUrl'][0])
+        for rendition in info['Renditions']:
+            if rendition['RenditionType'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(rendition['Url'], video_id, m3u8_id='hls'))
+            elif rendition['RenditionType'] == 'aac':
+                continue
+            else:
+                rendition_url = compat_urlparse.urlunparse(parsed_video_url._replace(path=replace_extension(parsed_video_url.path.replace('//', '/%s/' % rendition['ID']), rendition['RenditionType'])))
+                quality = self._QUALITIES.get(rendition['ID'], {})
                 formats.append({
-                    'format_id': compat_str(quality),
-                    'url': 'http://avideos.5min.com/%s/%s/%s_%s.mp4' % (second_id[-3:], second_id, video_id, quality),
-                    'height': height,
+                    'format_id': '%s-%d' % (rendition['RenditionType'], rendition['ID']),
+                    'url': rendition_url,
+                    'width': quality.get('width'),
+                    'height': quality.get('height'),
                 })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': info['Title'],
+            'thumbnail': info.get('ThumbURL'),
+            'duration': parse_duration(info.get('Duration')),
             'formats': formats,
         }

From 1f36085df94c2addd1175e7e299f6235aca3ac68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 14 Oct 2015 13:41:39 +0200
Subject: [PATCH 1573/2145] [vimeo] Fix extraction of password protected videos
 (fixes #7169)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 027f47ee3..fa1b22049 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -212,7 +212,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_v=1; vuid=%s' % vuid)
+        password_request.add_header('Cookie', 'clip_test2=1; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,

From 36bb63fad19df5ee419979f875e2265936511644 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 14:13:53 +0100
Subject: [PATCH 1574/2145] [criterion] fix description extraction

---
 youtube_dl/extractor/criterion.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py
index 4fb178165..dedb810a0 100644
--- a/youtube_dl/extractor/criterion.py
+++ b/youtube_dl/extractor/criterion.py
@@ -27,9 +27,7 @@ class CriterionIE(InfoExtractor):
         final_url = self._search_regex(
             r'so.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
         title = self._og_search_title(webpage)
-        description = self._html_search_regex(
-            r'<meta name="description" content="(.+?)" />',
-            webpage, 'video description')
+        description = self._html_search_meta('description', webpage)
         thumbnail = self._search_regex(
             r'so.addVariable\("thumbnailURL", "(.+?)"\)\;',
             webpage, 'thumbnail url')

From 7a6d76a64d8a89a08bb79791506fc18b993c4580 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 20:49:39 +0600
Subject: [PATCH 1575/2145] [extractor/common] Require closing quote in
 _og_regexes (Closes #7174)

E.g. do not match `property='og:video:type'` when `og:video` is requested.
---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0082a4c84..a0c4af92f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -646,7 +646,8 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
-        property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
+        property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
+                       % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
             template % (property_re, content_re),

From 1c29e81e620241b9013b23e7acd9d6ab06587fb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 20:58:52 +0600
Subject: [PATCH 1576/2145] [test_InfoExtractor] Add test for
 7a6d76a64d8a89a08bb79791506fc18b993c4580

---
 test/test_InfoExtractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index be8d12997..4ce5b5a35 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -35,10 +35,12 @@ class TestInfoExtractor(unittest.TestCase):
             <meta name="og:title" content='Foo'/>
             <meta content="Some video's description " name="og:description"/>
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
+            <meta content='application/x-shockwave-flash' property='og:video:type'>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
+        self.assertEqual(ie._og_search_video_url(html, default=None), None)
 
     def test_html_search_meta(self):
         ie = self.ie

From db0a8ad97993cb3f0c398d3a5dc55389565e0ffd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 21:11:06 +0600
Subject: [PATCH 1577/2145] [test_InfoExtractor] Add test for unquoted
 attribute

---
 test/test_InfoExtractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 4ce5b5a35..2a00d09a5 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -36,11 +36,13 @@ class TestInfoExtractor(unittest.TestCase):
             <meta content="Some video's description " name="og:description"/>
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
             <meta content='application/x-shockwave-flash' property='og:video:type'>
+            <meta content='Foo' property=og:foobar>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
         self.assertEqual(ie._og_search_video_url(html, default=None), None)
+        self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
 
     def test_html_search_meta(self):
         ie = self.ie

From ab953c64a0e8b8558e95d0318110c0885a4eec3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 21:15:29 +0600
Subject: [PATCH 1578/2145] [yandexmusic:track] Extract original size thumbnail
 (Closes #7160)

---
 youtube_dl/extractor/yandexmusic.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 6842f834f..08dc81f3a 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -46,14 +46,12 @@ class YandexMusicTrackIE(InfoExtractor):
                 % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
     def _get_track_info(self, track):
-        album = track['albums'][0]
-        a_thumb = None
-
-        if 'coverUri' in album:
-            a_thumb = album['coverUri']
-            if a_thumb:
-                a_thumb = 'http://' + a_thumb.replace('%%', '1000x1000')
-
+        thumbnail = None
+        cover_uri = track.get('albums', [{}])[0].get('coverUri')
+        if cover_uri:
+            thumbnail = cover_uri.replace('%%', 'orig')
+            if not thumbnail.startswith('http'):
+                thumbnail = 'http://' + thumbnail
         return {
             'id': track['id'],
             'ext': 'mp3',
@@ -61,7 +59,7 @@ class YandexMusicTrackIE(InfoExtractor):
             'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
-            'thumbnail': a_thumb,
+            'thumbnail': thumbnail,
         }
 
     def _real_extract(self, url):

From 9fb66c780cee8668b1bb07f70e70ae1161e13320 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 17:25:07 +0100
Subject: [PATCH 1579/2145] [megavideoz] remove extractor

---
 youtube_dl/extractor/__init__.py   |  1 -
 youtube_dl/extractor/megavideoz.py | 56 ------------------------------
 2 files changed, 57 deletions(-)
 delete mode 100644 youtube_dl/extractor/megavideoz.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f6d185818..462717b1e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -319,7 +319,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideoz import MegaVideozIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
deleted file mode 100644
index af7ff07ea..000000000
--- a/youtube_dl/extractor/megavideoz.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    xpath_text,
-)
-
-
-class MegaVideozIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
-    _TEST = {
-        'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
-        'info_dict': {
-            'id': '48723',
-            'display_id': 'SMPTE-Universal-Film-Leader',
-            'ext': 'mp4',
-            'title': 'SMPTE Universal Film Leader',
-            'thumbnail': 're:https?://.*?\.jpg',
-            'duration': 10.93,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
-
-        webpage = self._download_webpage(url, display_id)
-
-        if any(p in webpage for p in ('>Video Not Found<', '>404 Error<')):
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        config = self._download_xml(
-            self._search_regex(
-                r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),
-            display_id)
-
-        video_url = xpath_text(config, './file', 'video url', fatal=True)
-        title = xpath_text(config, './title', 'title', fatal=True)
-        thumbnail = xpath_text(config, './image', 'thumbnail')
-        duration = float_or_none(xpath_text(config, './duration', 'duration'))
-        video_id = xpath_text(config, './mediaid', 'video id') or video_id
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration
-        }

From 1812afb7b396f4954d5d1ca1cec1c3f2d67550c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:35:01 +0600
Subject: [PATCH 1580/2145] [utils] Do not fail in int_or_none on non-numeric
 data (Closes #7175)

---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1dc3153fd..86c693358 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1371,7 +1371,12 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
             v = getattr(v, get_attr, None)
     if v == '':
         v = None
-    return default if v is None else (int(v) * invscale // scale)
+    if v is None:
+        return default
+    try:
+        return int(v) * invscale // scale
+    except ValueError:
+        pass
 
 
 def str_or_none(v, default=None):

From caf80631f0c57b29187e2aa909fa1a3a6325d6e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:36:37 +0600
Subject: [PATCH 1581/2145] [utils] Do not fail in float_or_none on non-numeric
 data

---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86c693358..83b44caaa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1392,7 +1392,12 @@ def str_to_int(int_str):
 
 
 def float_or_none(v, scale=1, invscale=1, default=None):
-    return default if v is None else (float(v) * invscale / scale)
+    if v is None:
+        return default
+    try:
+        return float(v) * invscale / scale
+    except ValueError:
+        return default
 
 
 def parse_duration(s):

From af98f8ff37b3a0d9d1f743f4fc6c646333501eb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:37:03 +0600
Subject: [PATCH 1582/2145] [utils] Return default on fail in int_or_none

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 83b44caaa..7dbe25661 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1376,7 +1376,7 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     try:
         return int(v) * invscale // scale
     except ValueError:
-        pass
+        return default
 
 
 def str_or_none(v, default=None):

From 1db82381e38181aafbd78c65c58f005ad84cc08a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:52:25 +0600
Subject: [PATCH 1583/2145] [channel9] Add low quality formats and modernize

---
 youtube_dl/extractor/channel9.py | 35 ++++++++++++--------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 1ce004932..3a88181d8 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    parse_filesize,
+    qualities,
+)
 
 
 class Channel9IE(InfoExtractor):
@@ -52,23 +56,6 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
-    # Sorted by quality
-    _known_formats = ['MP3', 'MP4', 'Mid Quality WMV', 'Mid Quality MP4', 'High Quality WMV', 'High Quality MP4']
-
-    def _restore_bytes(self, formatted_size):
-        if not formatted_size:
-            return 0
-        m = re.match(r'^(?P<size>\d+(?:\.\d+)?)\s+(?P<units>[a-zA-Z]+)', formatted_size)
-        if not m:
-            return 0
-        units = m.group('units')
-        try:
-            exponent = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'].index(units.upper())
-        except ValueError:
-            return 0
-        size = float(m.group('size'))
-        return int(size * (1024 ** exponent))
-
     def _formats_from_html(self, html):
         FORMAT_REGEX = r'''
             (?x)
@@ -78,16 +65,20 @@ class Channel9IE(InfoExtractor):
             <h3>File\s+size</h3>\s*(?P<filesize>.*?)\s*
             </div>)?                                                # File size part may be missing
         '''
-        # Extract known formats
+        quality = qualities((
+            'MP3', 'MP4',
+            'Low Quality WMV', 'Low Quality MP4',
+            'Mid Quality WMV', 'Mid Quality MP4',
+            'High Quality WMV', 'High Quality MP4'))
         formats = [{
             'url': x.group('url'),
             'format_id': x.group('quality'),
             'format_note': x.group('note'),
             'format': '%s (%s)' % (x.group('quality'), x.group('note')),
-            'filesize': self._restore_bytes(x.group('filesize')),  # File size is approximate
-            'preference': self._known_formats.index(x.group('quality')),
+            'filesize_approx': parse_filesize(x.group('filesize')),
+            'quality': quality(x.group('quality')),
             'vcodec': 'none' if x.group('note') == 'Audio only' else None,
-        } for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats]
+        } for x in list(re.finditer(FORMAT_REGEX, html))]
 
         self._sort_formats(formats)
 

From a13d06de420f6968425d48030c37e1150ff9ed6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:57:59 +0600
Subject: [PATCH 1584/2145] [channel9] Add test for low quality mp4

---
 youtube_dl/extractor/channel9.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 3a88181d8..554399787 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -51,6 +51,21 @@ class Channel9IE(InfoExtractor):
                 'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
                 'authors': ['Mike Wilmot'],
             },
+        },
+        {
+            # low quality mp4 is best
+            'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+            'info_dict': {
+                'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+                'ext': 'mp4',
+                'title': 'Ranges for the Standard Library',
+                'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
+                'duration': 5646,
+                'thumbnail': 're:http://.*\.jpg',
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 

From fafc7950e2230bf25ac7c7563f1704cf8f134f64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:59:11 +0600
Subject: [PATCH 1585/2145] [channel9] Update tests' thumbnails

---
 youtube_dl/extractor/channel9.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 554399787..c74553dcf 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -32,7 +32,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Developer Kick-Off Session: Stuff We Love',
                 'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
                 'duration': 4576,
-                'thumbnail': 'http://video.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
+                'thumbnail': 're:http://.*\.jpg',
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
@@ -48,7 +48,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Self-service BI with Power BI - nuclear testing',
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
-                'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
+                'thumbnail': 're:http://.*\.jpg',
                 'authors': ['Mike Wilmot'],
             },
         },

From 90bddb6cdd59107d137c13970dc50a6193d204a7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 15 Oct 2015 14:28:56 +0100
Subject: [PATCH 1586/2145] [ooyala] extract more formats and metadata

---
 youtube_dl/extractor/ooyala.py | 151 ++++++++++++---------------------
 1 file changed, 53 insertions(+), 98 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index a262a9f6d..592cdc564 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -1,108 +1,64 @@
 from __future__ import unicode_literals
 import re
-import json
 import base64
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML,
-    ExtractorError,
-    determine_ext,
     int_or_none,
+    float_or_none,
 )
 
 
 class OoyalaBaseIE(InfoExtractor):
 
-    def _extract_result(self, info, more_info):
-        embedCode = info['embedCode']
-        video_url = info.get('ipad_url') or info['url']
-
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': 'mp4',
-            }]
-
-        return {
-            'id': embedCode,
-            'title': unescapeHTML(info['title']),
-            'formats': formats,
-            'description': unescapeHTML(more_info['description']),
-            'thumbnail': more_info['promo'],
+    def _extract(self, player_url, video_id):
+        print(player_url)
+        content_tree = self._download_json(player_url, video_id)['content_tree']
+        metadata = content_tree[list(content_tree)[0]]
+        embed_code = metadata['embed_code']
+        pcode = metadata.get('asset_pcode') or embed_code
+        video_info = {
+            'id': embed_code,
+            'title': metadata['title'],
+            'description': metadata.get('description'),
+            'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
+            'duration': int_or_none(metadata.get('duration')),
         }
 
-    def _extract(self, player_url, video_id):
-        player = self._download_webpage(player_url, video_id)
-        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
-                                        player, 'mobile player url')
-        # Looks like some videos are only available for particular devices
-        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
-        # is only available for ipad)
-        # Working around with fetching URLs for all the devices found starting with 'unknown'
-        # until we succeed or eventually fail for each device.
-        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
-        devices.remove('unknown')
-        devices.insert(0, 'unknown')
-        for device in devices:
-            mobile_player = self._download_webpage(
-                '%s&device=%s' % (mobile_url, device), video_id,
-                'Downloading mobile player JS for %s device' % device)
-            videos_info = self._search_regex(
-                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
-                mobile_player, 'info', fatal=False, default=None)
-            if videos_info:
-                break
-
-        if not videos_info:
-            formats = []
+        formats = []
+        for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (video_id, video_id),
-                video_id)
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=%s' % (pcode, embed_code, supported_format),
+                video_id, 'Downloading %s JSON' % supported_format)
 
-            cur_auth_data = auth_data['authorization_data'][video_id]
+            cur_auth_data = auth_data['authorization_data'][embed_code]
 
             for stream in cur_auth_data['streams']:
-                formats.append({
-                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
-                    'ext': stream.get('delivery_type'),
-                    'format': stream.get('video_codec'),
-                    'format_id': stream.get('profile'),
-                    'width': int_or_none(stream.get('width')),
-                    'height': int_or_none(stream.get('height')),
-                    'abr': int_or_none(stream.get('audio_bitrate')),
-                    'vbr': int_or_none(stream.get('video_bitrate')),
-                })
-            if formats:
-                return {
-                    'id': video_id,
-                    'formats': formats,
-                    'title': 'Ooyala video',
-                }
+                url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                delivery_type = stream['delivery_type']
+                if delivery_type == 'remote_asset':
+                    video_info['url'] = url
+                    return video_info
+                if delivery_type == 'hls':
+                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', 0, m3u8_id='hls', fatal=False))
+                elif delivery_type == 'hds':
+                    formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
+                else:
+                    formats.append({
+                        'url': url,
+                        'ext': stream.get('delivery_type'),
+                        'vcodec': stream.get('video_codec'),
+                        'format_id': stream.get('profile'),
+                        'width': int_or_none(stream.get('width')),
+                        'height': int_or_none(stream.get('height')),
+                        'abr': int_or_none(stream.get('audio_bitrate')),
+                        'vbr': int_or_none(stream.get('video_bitrate')),
+                        'fps': float_or_none(stream.get('framerate')),
+                    })
+        self._sort_formats(formats)
 
-            if not cur_auth_data['authorized']:
-                raise ExtractorError(cur_auth_data['message'], expected=True)
-
-        if not videos_info:
-            raise ExtractorError('Unable to extract info')
-        videos_info = videos_info.replace('\\"', '"')
-        videos_more_info = self._search_regex(
-            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
-        videos_info = json.loads(videos_info)
-        videos_more_info = json.loads(videos_more_info)
-
-        if videos_more_info.get('lineup'):
-            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
-            return {
-                '_type': 'playlist',
-                'id': video_id,
-                'title': unescapeHTML(videos_more_info['title']),
-                'entries': videos,
-            }
-        else:
-            return self._extract_result(videos_info[0], videos_more_info)
+        video_info['formats'] = formats
+        return video_info
 
 
 class OoyalaIE(OoyalaBaseIE):
@@ -117,6 +73,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'ext': 'mp4',
                 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
                 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
+                'duration': 853386,
             },
         }, {
             # Only available for ipad
@@ -125,7 +82,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                 'ext': 'mp4',
                 'title': 'Simulation Overview - Levels of Simulation',
-                'description': '',
+                'duration': 194948,
             },
         },
         {
@@ -136,7 +93,8 @@ class OoyalaIE(OoyalaBaseIE):
             'info_dict': {
                 'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
                 'ext': 'mp4',
-                'title': 'Ooyala video',
+                'title': 'Divide Tool Path.mp4',
+                'duration': 204405,
             }
         }
     ]
@@ -152,8 +110,8 @@ class OoyalaIE(OoyalaBaseIE):
 
     def _real_extract(self, url):
         embed_code = self._match_id(url)
-        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
-        return self._extract(player_url, embed_code)
+        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code)
+        return self._extract(content_tree_url, embed_code)
 
 
 class OoyalaExternalIE(OoyalaBaseIE):
@@ -170,7 +128,7 @@ class OoyalaExternalIE(OoyalaBaseIE):
                         .*?&pcode=
                     )
                     (?P<pcode>.+?)
-                    (&|$)
+                    (?:&|$)
                     '''
 
     _TEST = {
@@ -179,7 +137,7 @@ class OoyalaExternalIE(OoyalaBaseIE):
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'description': '',
+            'duration': 1302000,
         },
         'params': {
             # m3u8 download
@@ -188,9 +146,6 @@ class OoyalaExternalIE(OoyalaBaseIE):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        partner_id = mobj.group('partner_id')
-        video_id = mobj.group('id')
-        pcode = mobj.group('pcode')
-        player_url = 'http://player.ooyala.com/player.js?externalId=%s:%s&pcode=%s' % (partner_id, video_id, pcode)
-        return self._extract(player_url, video_id)
+        partner_id, video_id, pcode = re.match(self._VALID_URL, url).groups()
+        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/external_id/%s/%s:%s' % (pcode, partner_id, video_id)
+        return self._extract(content_tree_url, video_id)

From 497ca088a60fdd0a98f16e22a9d4fec135a26ab0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 15 Oct 2015 14:37:05 +0100
Subject: [PATCH 1587/2145] [ooyala] remove print statment

---
 youtube_dl/extractor/ooyala.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 592cdc564..df99a39f4 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -12,7 +12,6 @@ from ..utils import (
 class OoyalaBaseIE(InfoExtractor):
 
     def _extract(self, player_url, video_id):
-        print(player_url)
         content_tree = self._download_json(player_url, video_id)['content_tree']
         metadata = content_tree[list(content_tree)[0]]
         embed_code = metadata['embed_code']

From 6744f36db710eebe2ccc633e7f4f6132b968b0ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Oct 2015 08:44:19 +0600
Subject: [PATCH 1588/2145] [jeuxvideo] Fallback on og:title (Closes #7186,
 closes #7190)

---
 youtube_dl/extractor/jeuxvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 1df084d87..eef7daa29 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -28,7 +28,7 @@ class JeuxVideoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group(1)
         webpage = self._download_webpage(url, title)
-        title = self._html_search_meta('name', webpage)
+        title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
             r'data-src="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')

From dd414c970bcc493358ff6a76f6544a0417125594 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 16 Oct 2015 10:12:42 +0100
Subject: [PATCH 1589/2145] [ooyala] fix sorting and format id

---
 youtube_dl/extractor/ooyala.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index df99a39f4..075b594ce 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -39,15 +39,15 @@ class OoyalaBaseIE(InfoExtractor):
                     video_info['url'] = url
                     return video_info
                 if delivery_type == 'hls':
-                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', 0, m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                 elif delivery_type == 'hds':
-                    formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
+                    formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
                 else:
                     formats.append({
                         'url': url,
                         'ext': stream.get('delivery_type'),
                         'vcodec': stream.get('video_codec'),
-                        'format_id': stream.get('profile'),
+                        'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
                         'width': int_or_none(stream.get('width')),
                         'height': int_or_none(stream.get('height')),
                         'abr': int_or_none(stream.get('audio_bitrate')),

From cce9d15d0115e8b4cd1f6e2a327b5e9dbdf0ee54 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 16 Oct 2015 16:02:40 +0100
Subject: [PATCH 1590/2145] [ooyala] extract domain,handle errors and change
 related tests

---
 youtube_dl/extractor/byutv.py           |  5 ++-
 youtube_dl/extractor/generic.py         |  9 ++--
 youtube_dl/extractor/groupon.py         |  2 +
 youtube_dl/extractor/howcast.py         |  1 +
 youtube_dl/extractor/ooyala.py          | 60 ++++++++++++++-----------
 youtube_dl/extractor/teachingchannel.py |  1 +
 youtube_dl/extractor/vice.py            |  1 +
 7 files changed, 48 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 3b2de517e..ce25816f0 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -14,9 +14,10 @@ class BYUtvIE(InfoExtractor):
         'info_dict': {
             'id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
-            'description': 'md5:5438d33774b6bdc662f9485a340401cc',
+            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
             'title': 'Season 5 Episode 5',
-            'thumbnail': 're:^https?://.*\.jpg$'
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1486486,
         },
         'params': {
             'skip_download': True,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..805677364 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -335,6 +335,7 @@ class GenericIE(InfoExtractor):
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                 'ext': 'mp4',
                 'title': '2cc213299525360.mov',  # that's what we get
+                'duration': 238231,
             },
             'add_ie': ['Ooyala'],
         },
@@ -346,6 +347,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': '"Steve Jobs: Man in the Machine" trailer',
                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
+                'duration': 135427,
             },
             'params': {
                 'skip_download': True,
@@ -943,8 +945,9 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
                 'ext': 'mp4',
-                'description': 'VIDEO: Index/Match versus VLOOKUP.',
+                'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
                 'title': 'This is what separates the Excel masters from the wannabes',
+                'duration': 191933,
             },
             'params': {
                 # m3u8 downloads
@@ -1454,7 +1457,7 @@ class GenericIE(InfoExtractor):
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
-            return OoyalaIE._build_url_result(mobj.group('ec'))
+            return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
 
         # Look for multiple Ooyala embeds on SBN network websites
         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
@@ -1462,7 +1465,7 @@ class GenericIE(InfoExtractor):
             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
             if embeds:
                 return _playlist_from_matches(
-                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
+                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
 
         # Look for Aparat videos
         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 8b9e0e2f8..22ff7182f 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -18,6 +18,8 @@ class GrouponIE(InfoExtractor):
                 'id': 'tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
                 'ext': 'mp4',
                 'title': 'Bikram Yoga Huntington Beach | Orange County',
+                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+                'duration': 44961,
             },
         }],
         'params': {
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 16677f179..165b9f39e 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -16,6 +16,7 @@ class HowcastIE(InfoExtractor):
             'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
             'timestamp': 1276081287,
             'upload_date': '20100609',
+            'duration': 56823,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 075b594ce..3b692e903 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -6,13 +6,16 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     float_or_none,
+    ExtractorError,
+    unsmuggle_url,
 )
+from ..compat import compat_urllib_parse
 
 
 class OoyalaBaseIE(InfoExtractor):
 
-    def _extract(self, player_url, video_id):
-        content_tree = self._download_json(player_url, video_id)['content_tree']
+    def _extract(self, content_tree_url, video_id, domain='example.org'):
+        content_tree = self._download_json(content_tree_url, video_id)['content_tree']
         metadata = content_tree[list(content_tree)[0]]
         embed_code = metadata['embed_code']
         pcode = metadata.get('asset_pcode') or embed_code
@@ -27,33 +30,36 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=%s' % (pcode, embed_code, supported_format),
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?' % (pcode, embed_code) + compat_urllib_parse.urlencode({'domain': domain, 'supportedFormats': supported_format}),
                 video_id, 'Downloading %s JSON' % supported_format)
 
             cur_auth_data = auth_data['authorization_data'][embed_code]
 
-            for stream in cur_auth_data['streams']:
-                url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
-                delivery_type = stream['delivery_type']
-                if delivery_type == 'remote_asset':
-                    video_info['url'] = url
-                    return video_info
-                if delivery_type == 'hls':
-                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                elif delivery_type == 'hds':
-                    formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
-                else:
-                    formats.append({
-                        'url': url,
-                        'ext': stream.get('delivery_type'),
-                        'vcodec': stream.get('video_codec'),
-                        'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
-                        'width': int_or_none(stream.get('width')),
-                        'height': int_or_none(stream.get('height')),
-                        'abr': int_or_none(stream.get('audio_bitrate')),
-                        'vbr': int_or_none(stream.get('video_bitrate')),
-                        'fps': float_or_none(stream.get('framerate')),
-                    })
+            if cur_auth_data['authorized']:
+                for stream in cur_auth_data['streams']:
+                    url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                    delivery_type = stream['delivery_type']
+                    if delivery_type == 'remote_asset':
+                        video_info['url'] = url
+                        return video_info
+                    if delivery_type == 'hls':
+                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                    elif delivery_type == 'hds':
+                        formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
+                    else:
+                        formats.append({
+                            'url': url,
+                            'ext': stream.get('delivery_type'),
+                            'vcodec': stream.get('video_codec'),
+                            'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
+                            'width': int_or_none(stream.get('width')),
+                            'height': int_or_none(stream.get('height')),
+                            'abr': int_or_none(stream.get('audio_bitrate')),
+                            'vbr': int_or_none(stream.get('video_bitrate')),
+                            'fps': float_or_none(stream.get('framerate')),
+                        })
+            else:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, cur_auth_data['message']), expected=True)
         self._sort_formats(formats)
 
         video_info['formats'] = formats
@@ -108,9 +114,11 @@ class OoyalaIE(OoyalaBaseIE):
                               ie=cls.ie_key())
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         embed_code = self._match_id(url)
+        domain = smuggled_data.get('domain')
         content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code)
-        return self._extract(content_tree_url, embed_code)
+        return self._extract(content_tree_url, embed_code, domain)
 
 
 class OoyalaExternalIE(OoyalaBaseIE):
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index 117afa9bf..36a6fc679 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -16,6 +16,7 @@ class TeachingChannelIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'A History of Teaming',
             'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
+            'duration': 422255,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 01af7a995..7df87c31c 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -15,6 +15,7 @@ class ViceIE(InfoExtractor):
                 'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
                 'ext': 'mp4',
                 'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+                'duration': 725983,
             },
             'params': {
                 # Requires ffmpeg (m3u8 manifest)

From 8daeeedc06f420e2a87ba4755b56e721391cedba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:26:45 +0600
Subject: [PATCH 1591/2145] [bbc] Fix FutureWarning

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 68995f81e..1b3a33e4e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -421,7 +421,7 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
             description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
-            description = description_el.text if description_el else None
+            description = description_el.text if description_el is not None else None
 
             def get_programme_id(item):
                 def get_from_attributes(item):

From 1e52776ac3ebbafc2ec4697f3bc6ba05b7e5a9f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:46:38 +0600
Subject: [PATCH 1592/2145] [bandcamp] Prepend download URL with scheme when
 necessary (2) (#7077)

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index a27f3e748..f19e19001 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -52,7 +52,7 @@ class BandcampIE(InfoExtractor):
                     ext, abr_str = format_id.split('-', 1)
                     formats.append({
                         'format_id': format_id,
-                        'url': format_url,
+                        'url': self._proto_relative_url(format_url, 'http:'),
                         'ext': ext,
                         'vcodec': 'none',
                         'acodec': ext,

From ba717dca97925a21870fedcb46358d06cd5485ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:51:35 +0600
Subject: [PATCH 1593/2145] [bandcamp] Modernize

---
 youtube_dl/extractor/bandcamp.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index f19e19001..c1ef8051d 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -10,6 +10,8 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    int_or_none,
 )
 
 
@@ -56,7 +58,7 @@ class BandcampIE(InfoExtractor):
                         'ext': ext,
                         'vcodec': 'none',
                         'acodec': ext,
-                        'abr': int(abr_str),
+                        'abr': int_or_none(abr_str),
                     })
 
                 self._sort_formats(formats)
@@ -65,7 +67,7 @@ class BandcampIE(InfoExtractor):
                     'id': compat_str(data['id']),
                     'title': data['title'],
                     'formats': formats,
-                    'duration': float(data['duration']),
+                    'duration': float_or_none(data.get('duration')),
                 }
             else:
                 raise ExtractorError('No free songs found')

From 246ce1085804ead9126328cb1ec761d308f561c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 01:08:23 +0600
Subject: [PATCH 1594/2145] [rte] Update _VALID_URL (Closes #7198)

---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 04158b993..427c70866 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/(?P<id>[0-9]+)/'
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/de/show/10363114/',
         'info_dict': {

From 2ccb37beb9e35ebbf2cdf65a4c1641e5286de1e8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Oct 2015 21:40:38 +0200
Subject: [PATCH 1595/2145] release 2015.10.16

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dc0354095..47f7da86d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -159,6 +159,7 @@
  - **facebook**
  - **faz.net**
  - **fc2**
+ - **Fczenit**
  - **fernsehkritik.tv**
  - **Firstpost**
  - **FiveTV**
@@ -281,7 +282,6 @@
  - **Malemotion**
  - **MDR**
  - **media.ccc.de**
- - **MegaVideoz**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index aaa43d315..31d2a9dc0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.13'
+__version__ = '2015.10.16'

From 36eb802baffda9930e8c821e1adf94b0b53b5ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 17 Oct 2015 11:49:51 +0200
Subject: [PATCH 1596/2145] [rte] Replace expired test

According to their webpage it should be available until October 2035.
---
 youtube_dl/extractor/rte.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 427c70866..2811767b9 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -11,14 +11,14 @@ from ..utils import (
 class RteIE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://www.rte.ie/player/de/show/10363114/',
+        'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
         'info_dict': {
-            'id': '10363114',
+            'id': '10478715',
             'ext': 'mp4',
-            'title': 'One News',
+            'title': 'Watch iWitness  online',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'The One O\'Clock News followed by Weather.',
-            'duration': 436.844,
+            'description': 'iWitness : The spirit of Ireland, one voice and one minute at a time.',
+            'duration': 60.046,
         },
         'params': {
             'skip_download': 'f4m fails with --test atm'

From 6df7179e6c3c7df165b5788ecb5e712da122356f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 17 Oct 2015 11:53:59 +0200
Subject: [PATCH 1597/2145] [rte] Actually recognize https urls

There was a missing 's' before the '?'.
---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 2811767b9..d9cfbf180 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
         'info_dict': {

From fbd9f6ea804328d536aafd2b20a8afb72968e351 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 18:28:21 +0600
Subject: [PATCH 1598/2145] [twitch] Improve authentication

---
 youtube_dl/extractor/twitch.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 023911c41..891499a1f 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,6 +15,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     int_or_none,
     parse_duration,
@@ -27,8 +28,7 @@ class TwitchBaseIE(InfoExtractor):
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'https://secure.twitch.tv/login'
-    _LOGIN_POST_URL = 'https://passport.twitch.tv/authentications/new'
+    _LOGIN_URL = 'http://www.twitch.tv/login'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -61,26 +61,28 @@ class TwitchBaseIE(InfoExtractor):
         if username is None:
             return
 
-        login_page = self._download_webpage(
+        login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
 
         login_form = self._hidden_inputs(login_page)
 
         login_form.update({
-            'login': username.encode('utf-8'),
-            'password': password.encode('utf-8'),
+            'username': username,
+            'password': password,
         })
 
+        redirect_url = handle.geturl()
+
         post_url = self._search_regex(
             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
-            'post url', default=self._LOGIN_POST_URL, group='url')
+            'post url', default=redirect_url, group='url')
 
         if not post_url.startswith('http'):
-            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+            post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
         request = compat_urllib_request.Request(
-            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        request.add_header('Referer', self._LOGIN_URL)
+            post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
+        request.add_header('Referer', redirect_url)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From e5e9966199c00a6b89f1f25e1c7b85effb032537 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 18:29:54 +0600
Subject: [PATCH 1599/2145] [twitch:vod] Improve extraction

---
 youtube_dl/extractor/twitch.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 891499a1f..21ea836ea 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -240,14 +240,24 @@ class TwitchVodIE(TwitchItemBaseIE):
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
+
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
         access_token = self._download_json(
-            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-            'Downloading %s access token' % self._ITEM_TYPE)
+           '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+           'Downloading %s access token' % self._ITEM_TYPE)
+
         formats = self._extract_m3u8_formats(
-            '%s/vod/%s?nauth=%s&nauthsig=%s&allow_source=true'
-            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
-            item_id, 'mp4')
+           '%s/vod/%s?%s' % (
+               self._USHER_BASE, item_id,
+               compat_urllib_parse.urlencode({
+                    'allow_source': 'true',
+                    'allow_spectre': 'true',
+                    'player': 'twitchweb',
+                    'nauth': access_token['token'],
+                    'nauthsig': access_token['sig'],
+                })),
+           item_id, 'mp4')
+
         self._prefer_source(formats)
         info['formats'] = formats
 

From 350c9481336ac981eadc982b67ccdbc7e28ca0e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 18:43:12 +0600
Subject: [PATCH 1600/2145] [twitch:vod] Formatting

---
 youtube_dl/extractor/twitch.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 21ea836ea..3ec08b674 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -243,20 +243,20 @@ class TwitchVodIE(TwitchItemBaseIE):
 
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
         access_token = self._download_json(
-           '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-           'Downloading %s access token' % self._ITEM_TYPE)
+            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+            'Downloading %s access token' % self._ITEM_TYPE)
 
         formats = self._extract_m3u8_formats(
-           '%s/vod/%s?%s' % (
-               self._USHER_BASE, item_id,
-               compat_urllib_parse.urlencode({
+            '%s/vod/%s?%s' % (
+                self._USHER_BASE, item_id,
+                compat_urllib_parse.urlencode({
                     'allow_source': 'true',
                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'nauth': access_token['token'],
                     'nauthsig': access_token['sig'],
                 })),
-           item_id, 'mp4')
+            item_id, 'mp4')
 
         self._prefer_source(formats)
         info['formats'] = formats

From d90e40305bee84f5e3cd4927c729e1d16bbd3dc6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 17:28:09 +0100
Subject: [PATCH 1601/2145] [bilibili] fix info extraction

---
 youtube_dl/extractor/bilibili.py | 151 ++++++++++++-------------------
 1 file changed, 57 insertions(+), 94 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index ecc17ebeb..8f23a30ab 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,15 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import itertools
 import json
 import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    unified_strdate,
     ExtractorError,
 )
 
@@ -21,12 +18,15 @@ class BiliBiliIE(InfoExtractor):
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '2c301e4dab317596e837c3e7633e7d86',
         'info_dict': {
-            'id': '1074402_part1',
+            'id': '1554319',
             'ext': 'flv',
             'title': '【金坷垃】金泡沫',
-            'duration': 308,
+            'duration': 308313,
             'upload_date': '20140420',
             'thumbnail': 're:^https?://.+\.jpg',
+            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
+            'timestamp': 1397983878,
+            'uploader': '菊子桑',
         },
     }, {
         'url': 'http://www.bilibili.com/video/av1041170/',
@@ -34,109 +34,72 @@ class BiliBiliIE(InfoExtractor):
             'id': '1041170',
             'title': '【BD1080P】刀语【诸神&异域】',
         },
-        'playlist_count': 9,
+        'playlist_count': 12,
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+    def _extract_video_info(self, cid, view_data, page_num=1, num_pages=1):
+        title = view_data['title']
 
-        if '(此视频不存在或被删除)' in webpage:
-            raise ExtractorError(
-                'The video does not exist or was deleted', expected=True)
-
-        if '>你没有权限浏览！ 由于版权相关问题 我们不对您所在的地区提供服务<' in webpage:
-            raise ExtractorError(
-                'The video is not available in your region due to copyright reasons',
-                expected=True)
-
-        video_code = self._search_regex(
-            r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
-
-        title = self._html_search_meta(
-            'media:title', video_code, 'title', fatal=True)
-        duration_str = self._html_search_meta(
-            'duration', video_code, 'duration')
-        if duration_str is None:
-            duration = None
-        else:
-            duration_mobj = re.match(
-                r'^T(?:(?P<hours>[0-9]+)H)?(?P<minutes>[0-9]+)M(?P<seconds>[0-9]+)S$',
-                duration_str)
-            duration = (
-                int_or_none(duration_mobj.group('hours'), default=0) * 3600 +
-                int(duration_mobj.group('minutes')) * 60 +
-                int(duration_mobj.group('seconds')))
-        upload_date = unified_strdate(self._html_search_meta(
-            'uploadDate', video_code, fatal=False))
-        thumbnail = self._html_search_meta(
-            'thumbnailUrl', video_code, 'thumbnail', fatal=False)
-
-        cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
-
-        entries = []
-
-        lq_page = self._download_webpage(
-            'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
-            video_id,
-            note='Downloading LQ video info'
+        page = self._download_webpage(
+            'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
+            cid,
+            'Downloading page %d/%d' % (page_num, num_pages)
         )
         try:
-            err_info = json.loads(lq_page)
+            err_info = json.loads(page)
             raise ExtractorError(
                 'BiliBili said: ' + err_info['error_text'], expected=True)
         except ValueError:
             pass
 
-        lq_doc = ET.fromstring(lq_page)
-        lq_durls = lq_doc.findall('./durl')
+        doc = ET.fromstring(page)
+        durls = doc.findall('./durl')
 
-        hq_doc = self._download_xml(
-            'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
-            video_id,
-            note='Downloading HQ video info',
-            fatal=False,
-        )
-        if hq_doc is not False:
-            hq_durls = hq_doc.findall('./durl')
-            assert len(lq_durls) == len(hq_durls)
-        else:
-            hq_durls = itertools.repeat(None)
-
-        i = 1
-        for lq_durl, hq_durl in zip(lq_durls, hq_durls):
-            formats = [{
-                'format_id': 'lq',
-                'quality': 1,
-                'url': lq_durl.find('./url').text,
-                'filesize': int_or_none(
-                    lq_durl.find('./size'), get_attr='text'),
-            }]
-            if hq_durl is not None:
-                formats.append({
-                    'format_id': 'hq',
-                    'quality': 2,
-                    'ext': 'flv',
-                    'url': hq_durl.find('./url').text,
-                    'filesize': int_or_none(
-                        hq_durl.find('./size'), get_attr='text'),
-                })
-            self._sort_formats(formats)
+        entries = []
 
+        for durl in durls:
             entries.append({
-                'id': '%s_part%d' % (video_id, i),
+                'id': '%s_part%s' % (cid, durl.find('./order').text),
                 'title': title,
-                'formats': formats,
-                'duration': duration,
-                'upload_date': upload_date,
-                'thumbnail': thumbnail,
+                'url': durl.find('./url').text,
+                'filesize': int_or_none(durl.find('./filesize').text),
+                'ext': 'flv',
+                'duration': int_or_none(durl.find('./length').text) // 1000,
             })
 
-            i += 1
-
-        return {
-            '_type': 'multi_video',
-            'entries': entries,
-            'id': video_id,
-            'title': title
+        info = {
+            'id': cid,
+            'title': title,
+            'description': view_data.get('description'),
+            'thumbnail': view_data.get('pic'),
+            'uploader': view_data.get('author'),
+            'timestamp': int_or_none(view_data.get('created')),
+            'view_count': view_data.get('play'),
+            'duration': int_or_none(doc.find('./timelength').text),
         }
+
+        if len(entries) == 1:
+            entries[0].update(info)
+            return entries[0]
+        else:
+            info.update({
+                '_type': 'multi_video',
+                'entries': entries,
+            })
+            return info
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        view_data = self._download_json('http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s' % video_id, video_id)
+
+        num_pages = int_or_none(view_data['pages'])
+        if num_pages > 1:
+            play_list_title = view_data['title']
+            page_list = self._download_json('http://www.bilibili.com/widget/getPageList?aid=%s' % video_id, video_id, 'Downloading page list metadata')
+            entries = []
+            for page in page_list:
+                view_data['title'] = page['pagename']
+                entries.append(self._extract_video_info(str(page['cid']), view_data, page['page'], num_pages))
+            return self.playlist_result(entries, video_id, play_list_title, view_data.get('description'))
+        else:
+            return self._extract_video_info(str(view_data['cid']), view_data)

From 41a7b00f183844e93ae2ba46fb4021f257f3ce79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 18:18:40 +0200
Subject: [PATCH 1602/2145] [vimeo] Extract config URL from (new?) React-based
 Vimeo's page

---
 youtube_dl/extractor/vimeo.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fa1b22049..88e462a4d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -286,7 +286,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
         try:
             try:
                 config_url = self._html_search_regex(
-                    r' data-config-url="(.+?)"', webpage, 'config URL')
+                    r' data-config-url="(.+?)"', webpage,
+                    'config URL', default=None)
+                if not config_url:
+                    # New react-based page
+                    vimeo_clip_page_config = self._search_regex(
+                        r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
+                        'vimeo clip page config')
+                    config_url = self._parse_json(vimeo_clip_page_config, video_id)['player']['config_url']
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:

From dd8417526b13c541e6db8f4200e717b8922a1620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 22:48:14 +0600
Subject: [PATCH 1603/2145] [vimeo] Clarify new react+flux website fallback

---
 youtube_dl/extractor/vimeo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 88e462a4d..0f84656c0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -289,11 +289,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     r' data-config-url="(.+?)"', webpage,
                     'config URL', default=None)
                 if not config_url:
-                    # New react-based page
+                    # Sometimes new react-based page is served instead of old one that require
+                    # different config URL extraction approach (see
+                    # https://github.com/rg3/youtube-dl/pull/7209)
                     vimeo_clip_page_config = self._search_regex(
                         r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
                         'vimeo clip page config')
-                    config_url = self._parse_json(vimeo_clip_page_config, video_id)['player']['config_url']
+                    config_url = self._parse_json(
+                        vimeo_clip_page_config, video_id)['player']['config_url']
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:

From 59fe4824f80b7e266ea9918ae1b2e49a456b869f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 18:52:25 +0200
Subject: [PATCH 1604/2145] [vidme] Better error message for suspended vidme
 videos

---
 youtube_dl/extractor/vidme.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 078d283b2..81dcaa231 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -114,6 +114,12 @@ class VidmeIE(InfoExtractor):
 
         video = response['video']
 
+        if video.get('state') == 'user-disabled':
+            raise ExtractorError(
+                'Vidme said: This video has been suspended either due to a copyright claim, '
+                'or for violating the terms of use.',
+                expected=True)
+
         formats = [{
             'format_id': f.get('type'),
             'url': f['uri'],

From 9eb31b265f65ec6b04a508702af1a6feddafb8fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 23:01:24 +0600
Subject: [PATCH 1605/2145] [vidme] Add user-disabled test

---
 youtube_dl/extractor/vidme.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 81dcaa231..382517a4a 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -93,6 +93,10 @@ class VidmeIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # nsfw, user-disabled
+        'url': 'https://vid.me/dzGJ',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 583882fdce19f8c565402f42523b275f96c91575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 19:26:30 +0200
Subject: [PATCH 1606/2145] [dailymotion] Report errors from player v5

---
 youtube_dl/extractor/dailymotion.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 80a05cfee..ea1edceb1 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -96,6 +96,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader': 'HotWaves1012',
                 'age_limit': 18,
             }
+        },
+        # geo-restricted, player v5
+        {
+            'url': 'http://www.dailymotion.com/video/xhza0o',
+            'only_matching': True,
         }
     ]
 
@@ -124,6 +129,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         if player_v5:
             player = self._parse_json(player_v5, video_id)
             metadata = player['metadata']
+
+            self._check_error(metadata)
+
             formats = []
             for quality, media_list in metadata['qualities'].items():
                 for media in media_list:
@@ -201,9 +209,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'video info', flags=re.MULTILINE),
             video_id)
 
-        if info.get('error') is not None:
-            msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
-            raise ExtractorError(msg, expected=True)
+        self._check_error(info)
 
         formats = []
         for (key, format_id) in self._FORMATS:
@@ -246,6 +252,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'duration': info['duration']
         }
 
+    def _check_error(self, info):
+        if info.get('error') is not None:
+            msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
+            raise ExtractorError(msg, expected=True)
+
     def _get_subtitles(self, video_id, webpage):
         try:
             sub_list = self._download_webpage(

From 55af2b26e0f169bef2f10a7b5f6ec8e34c6dbb6d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 18:30:51 +0100
Subject: [PATCH 1607/2145] [bilibili] extract backup url

---
 youtube_dl/extractor/bilibili.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 8f23a30ab..85156ce49 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -58,13 +58,21 @@ class BiliBiliIE(InfoExtractor):
         entries = []
 
         for durl in durls:
+            formats = []
+            backup_url = durl.find('./backup_url')
+            if backup_url is not None:
+                formats.append({'url': backup_url.find('./url').text})
+            size = durl.find('./filesize|./size')
+            formats.append({
+                'url': durl.find('./url').text,
+                'filesize': int_or_none(size.text) if size else None,
+                'ext': 'flv',
+            })
             entries.append({
                 'id': '%s_part%s' % (cid, durl.find('./order').text),
                 'title': title,
-                'url': durl.find('./url').text,
-                'filesize': int_or_none(durl.find('./filesize').text),
-                'ext': 'flv',
                 'duration': int_or_none(durl.find('./length').text) // 1000,
+                'formats': formats,
             })
 
         info = {

From 648e6a1ffe45ceae2995c3f9ec6a9413aad55640 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 00:11:34 +0600
Subject: [PATCH 1608/2145] [youtube] Generalize playlist entries extraction
 (Closes #6699, closes #6992)

---
 youtube_dl/extractor/youtube.py | 121 ++++++++++++++------------------
 1 file changed, 52 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b252e36e1..08e821362 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,6 +178,52 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
+class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
+    # Extract the video ids from the playlist pages
+    def _entries(self, page, playlist_id):
+        more_widget_html = content_html = page
+        for page_num in itertools.count(1):
+            for video_id, video_title in self.extract_videos_from_page(content_html):
+                yield self.url_result(
+                    video_id, 'Youtube', video_id=video_id,
+                    video_title=video_title)
+
+            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+            if not mobj:
+                break
+
+            more = self._download_json(
+                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+                'Downloading page #%s' % page_num,
+                transform_source=uppercase_escape)
+            content_html = more['content_html']
+            if not content_html.strip():
+                # Some webpages show a "Load more" button but they don't
+                # have more videos
+                break
+            more_widget_html = more['load_more_widget_html']
+
+    def extract_videos_from_page(self, page):
+        ids_in_page = []
+        titles_in_page = []
+        for mobj in re.finditer(self._VIDEO_RE, page):
+            # The link with index 0 is not the first video of the playlist (not sure if still actual)
+            if 'index' in mobj.groupdict() and mobj.group('id') == '0':
+                continue
+            video_id = mobj.group('id')
+            video_title = unescapeHTML(mobj.group('title'))
+            if video_title:
+                video_title = video_title.strip()
+            try:
+                idx = ids_in_page.index(video_id)
+                if video_title and not titles_in_page[idx]:
+                    titles_in_page[idx] = video_title
+            except ValueError:
+                ids_in_page.append(video_id)
+                titles_in_page.append(video_title)
+        return zip(ids_in_page, titles_in_page)
+
+
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
     _VALID_URL = r"""(?x)^
@@ -1419,7 +1465,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
-class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
+class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:
                         (?:https?://)?
@@ -1440,7 +1486,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
-    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
+    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
     IE_NAME = 'youtube:playlist'
     _TESTS = [{
         'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
@@ -1557,37 +1603,11 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             else:
                 self.report_warning('Youtube gives an alert message: ' + match)
 
-        # Extract the video ids from the playlist pages
-        def _entries():
-            more_widget_html = content_html = page
-            for page_num in itertools.count(1):
-                matches = re.finditer(self._VIDEO_RE, content_html)
-                # We remove the duplicates and the link with index 0
-                # (it's not the first video of the playlist)
-                new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
-                for vid_id in new_ids:
-                    yield self.url_result(vid_id, 'Youtube', video_id=vid_id)
-
-                mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-                if not mobj:
-                    break
-
-                more = self._download_json(
-                    'https://youtube.com/%s' % mobj.group('more'), playlist_id,
-                    'Downloading page #%s' % page_num,
-                    transform_source=uppercase_escape)
-                content_html = more['content_html']
-                if not content_html.strip():
-                    # Some webpages show a "Load more" button but they don't
-                    # have more videos
-                    break
-                more_widget_html = more['load_more_widget_html']
-
         playlist_title = self._html_search_regex(
             r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
             page, 'title')
 
-        return self.playlist_result(_entries(), playlist_id, playlist_title)
+        return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
 
     def _real_extract(self, url):
         # Extract playlist id
@@ -1613,10 +1633,11 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         return self._extract_playlist(playlist_id)
 
 
-class YoutubeChannelIE(InfoExtractor):
+class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
     _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
+    _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
     IE_NAME = 'youtube:channel'
     _TESTS = [{
         'note': 'paginated channel',
@@ -1627,22 +1648,6 @@ class YoutubeChannelIE(InfoExtractor):
         }
     }]
 
-    @staticmethod
-    def extract_videos_from_page(page):
-        ids_in_page = []
-        titles_in_page = []
-        for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):
-            video_id = mobj.group('id')
-            video_title = unescapeHTML(mobj.group('title'))
-            try:
-                idx = ids_in_page.index(video_id)
-                if video_title and not titles_in_page[idx]:
-                    titles_in_page[idx] = video_title
-            except ValueError:
-                ids_in_page.append(video_id)
-                titles_in_page.append(video_title)
-        return zip(ids_in_page, titles_in_page)
-
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
@@ -1685,29 +1690,7 @@ class YoutubeChannelIE(InfoExtractor):
                 for video_id, video_title in self.extract_videos_from_page(channel_page)]
             return self.playlist_result(entries, channel_id)
 
-        def _entries():
-            more_widget_html = content_html = channel_page
-            for pagenum in itertools.count(1):
-
-                for video_id, video_title in self.extract_videos_from_page(content_html):
-                    yield self.url_result(
-                        video_id, 'Youtube', video_id=video_id,
-                        video_title=video_title)
-
-                mobj = re.search(
-                    r'data-uix-load-more-href="/?(?P<more>[^"]+)"',
-                    more_widget_html)
-                if not mobj:
-                    break
-
-                more = self._download_json(
-                    'https://youtube.com/%s' % mobj.group('more'), channel_id,
-                    'Downloading page #%s' % (pagenum + 1),
-                    transform_source=uppercase_escape)
-                content_html = more['content_html']
-                more_widget_html = more['load_more_widget_html']
-
-        return self.playlist_result(_entries(), channel_id)
+        return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
 
 
 class YoutubeUserIE(YoutubeChannelIE):

From 8e5b1219489be399de55566090e145c89007fa48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 00:27:06 +0600
Subject: [PATCH 1609/2145] [test_youtube_lists] Add test flat playlist
 entries' titles

---
 test/test_youtube_lists.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index c889b6f15..26aadb34f 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -57,5 +57,14 @@ class TestYoutubeLists(unittest.TestCase):
         entries = result['entries']
         self.assertEqual(len(entries), 100)
 
+    def test_youtube_flat_playlist_titles(self):
+        dl = FakeYDL()
+        dl.params['extract_flat'] = True
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        self.assertIsPlaylist(result)
+        for entry in result['entries']:
+            self.assertTrue(entry.get('title'))
+
 if __name__ == '__main__':
     unittest.main()

From 7593fbaa126f8bf14eecff7f103cb497e3d31de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 01:00:37 +0600
Subject: [PATCH 1610/2145] [dailymotion] Error spelling

---
 youtube_dl/extractor/dailymotion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index ea1edceb1..9cd9ff17d 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -254,8 +254,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _check_error(self, info):
         if info.get('error') is not None:
-            msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
-            raise ExtractorError(msg, expected=True)
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, info['error']['title']), expected=True)
 
     def _get_subtitles(self, video_id, webpage):
         try:

From 5a11b793fe70beb6b0c7a74a489db9e52c4a742b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 01:36:03 +0600
Subject: [PATCH 1611/2145] [lynda] Extract all prioritized streams

---
 youtube_dl/extractor/lynda.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 378117270..5c973e75c 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -140,13 +140,14 @@ class LyndaIE(LyndaBaseIE):
 
         prioritized_streams = video_json.get('PrioritizedStreams')
         if prioritized_streams:
-            formats.extend([
-                {
-                    'url': video_url,
-                    'width': int_or_none(format_id),
-                    'format_id': format_id,
-                } for format_id, video_url in prioritized_streams['0'].items()
-            ])
+            for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
+                formats.extend([
+                    {
+                        'url': video_url,
+                        'width': int_or_none(format_id),
+                        'format_id': '%s-%s' % (prioritized_stream_id, format_id),
+                    } for format_id, video_url in prioritized_stream.items()
+                ])
 
         self._check_formats(formats, video_id)
         self._sort_formats(formats)

From 355c7ad361aa3c8a57ff83e3f702a496dce59e65 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 21:30:38 +0100
Subject: [PATCH 1612/2145] [cspan] handle error massages and extract qualities

---
 youtube_dl/extractor/cspan.py | 67 +++++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 994e080d5..c74b35fd9 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -9,16 +9,21 @@ from ..utils import (
     find_xpath_attr,
     smuggle_url,
     determine_ext,
+    ExtractorError,
 )
 from .senateisvp import SenateISVPIE
 
 
+def get_text_attr(d, attr):
+    return d.get(attr, {}).get('#text')
+
+
 class CSpanIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
     IE_DESC = 'C-SPAN'
     _TESTS = [{
         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
-        'md5': '067803f994e049b455a58b16e5aab442',
+        'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
         'info_dict': {
             'id': '315139',
             'ext': 'mp4',
@@ -28,7 +33,7 @@ class CSpanIE(InfoExtractor):
         'skip': 'Regularly fails on travis, for unknown reasons',
     }, {
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
-        'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2',
+        'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
         'info_dict': {
             'id': 'c4486943',
             'ext': 'mp4',
@@ -37,7 +42,7 @@ class CSpanIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
-        'md5': '446562a736c6bf97118e389433ed88d4',
+        'md5': '2ae5051559169baadba13fc35345ae74',
         'info_dict': {
             'id': '342759',
             'ext': 'mp4',
@@ -71,8 +76,10 @@ class CSpanIE(InfoExtractor):
                 return self.url_result(surl, 'SenateISVP', video_id, title)
 
         data = self._download_json(
-            'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
-            video_id)
+            'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+            video_id)['video']
+        if data['@status'] != 'Success':
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
 
         doc = self._download_xml(
             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -83,28 +90,36 @@ class CSpanIE(InfoExtractor):
         title = find_xpath_attr(doc, './/string', 'name', 'title').text
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
-        files = data['video']['files']
-        try:
-            capfile = data['video']['capfile']['#text']
-        except KeyError:
-            capfile = None
+        files = data['files']
+        capfile = get_text_attr(data, 'capfile')
 
-        entries = [{
-            'id': '%s_%d' % (video_id, partnum + 1),
-            'title': (
-                title if len(files) == 1 else
-                '%s part %d' % (title, partnum + 1)),
-            'url': unescapeHTML(f['path']['#text']),
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': int_or_none(f.get('length', {}).get('#text')),
-            'subtitles': {
-                'en': [{
-                    'url': capfile,
-                    'ext': determine_ext(capfile, 'dfxp')
-                }],
-            } if capfile else None,
-        } for partnum, f in enumerate(files)]
+        entries = []
+        for partnum, f in enumerate(files):
+            formats = []
+            for quality in f['qualities']:
+                formats.append({
+                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(get_text_attr(quality, 'file')),
+                    'height': int_or_none(get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+                })
+            self._sort_formats(formats)
+            entries.append({
+                'id': '%s_%d' % (video_id, partnum + 1),
+                'title': (
+                    title if len(files) == 1 else
+                    '%s part %d' % (title, partnum + 1)),
+                'formats': formats,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': int_or_none(get_text_attr(f, 'length')),
+                'subtitles': {
+                    'en': [{
+                        'url': capfile,
+                        'ext': determine_ext(capfile, 'dfxp')
+                    }],
+                } if capfile else None,
+            })
 
         if len(entries) == 1:
             entry = dict(entries[0])

From 520e753390aab5845bb257b964ebcd6f818455df Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 23:12:58 +0100
Subject: [PATCH 1613/2145] [bilibili] add support for specefic page extraction

---
 youtube_dl/extractor/bilibili.py | 45 ++++++++++++++++----------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 85156ce49..e00efb6eb 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,18 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import json
 import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    unescapeHTML,
     ExtractorError,
 )
 
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>[0-9]+)/'
+    _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
@@ -33,17 +35,31 @@ class BiliBiliIE(InfoExtractor):
         'info_dict': {
             'id': '1041170',
             'title': '【BD1080P】刀语【诸神&异域】',
+            'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
+            'uploader': '枫叶逝去',
+            'timestamp': 1396501299,
         },
-        'playlist_count': 12,
+        'playlist_count': 9,
     }]
 
-    def _extract_video_info(self, cid, view_data, page_num=1, num_pages=1):
-        title = view_data['title']
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        page_num = mobj.group('page_num') or '1'
+
+        view_data = self._download_json(
+            'http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s&page=%s' % (video_id, page_num),
+            video_id)
+        if 'error' in view_data:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, view_data['error']), expected=True)
+
+        cid = view_data['cid']
+        title = unescapeHTML(view_data['title'])
 
         page = self._download_webpage(
             'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
             cid,
-            'Downloading page %d/%d' % (page_num, num_pages)
+            'Downloading page %s/%s' % (page_num, view_data['pages'])
         )
         try:
             err_info = json.loads(page)
@@ -76,7 +92,7 @@ class BiliBiliIE(InfoExtractor):
             })
 
         info = {
-            'id': cid,
+            'id': str(cid),
             'title': title,
             'description': view_data.get('description'),
             'thumbnail': view_data.get('pic'),
@@ -92,22 +108,7 @@ class BiliBiliIE(InfoExtractor):
         else:
             info.update({
                 '_type': 'multi_video',
+                'id': video_id,
                 'entries': entries,
             })
             return info
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        view_data = self._download_json('http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s' % video_id, video_id)
-
-        num_pages = int_or_none(view_data['pages'])
-        if num_pages > 1:
-            play_list_title = view_data['title']
-            page_list = self._download_json('http://www.bilibili.com/widget/getPageList?aid=%s' % video_id, video_id, 'Downloading page list metadata')
-            entries = []
-            for page in page_list:
-                view_data['title'] = page['pagename']
-                entries.append(self._extract_video_info(str(page['cid']), view_data, page['page'], num_pages))
-            return self.playlist_result(entries, video_id, play_list_title, view_data.get('description'))
-        else:
-            return self._extract_video_info(str(view_data['cid']), view_data)

From 80f48920c8a909ba55d13932524e55ed970f1c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 06:57:57 +0600
Subject: [PATCH 1614/2145] [crunchyroll] Bypass maturity wall (Closes #7202)

---
 youtube_dl/extractor/crunchyroll.py | 59 ++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 95952bc29..aa258bbc2 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -32,6 +32,26 @@ from ..aes import (
 
 
 class CrunchyrollBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'crunchyroll'
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        login_url = 'https://www.crunchyroll.com/?a=formhandler'
+        data = urlencode_postdata({
+            'formname': 'RpcApiUser_Login',
+            'name': username,
+            'password': password,
+        })
+        login_request = compat_urllib_request.Request(login_url, data)
+        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        self._download_webpage(login_request, None, False, 'Wrong login info')
+
+    def _real_initialize(self):
+        self._login()
+
     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                    else compat_urllib_request.Request(url_or_request))
@@ -46,10 +66,22 @@ class CrunchyrollBaseIE(InfoExtractor):
         return super(CrunchyrollBaseIE, self)._download_webpage(
             request, video_id, note, errnote, fatal, tries, timeout, encoding)
 
+    @staticmethod
+    def _add_skip_wall(url):
+        parsed_url = compat_urlparse.urlparse(url)
+        qs = compat_urlparse.parse_qs(parsed_url.query)
+        # Always force skip_wall to bypass maturity wall, namely 18+ confirmation message:
+        # > This content may be inappropriate for some people.
+        # > Are you sure you want to continue?
+        # since it's not disabled by default in crunchyroll account's settings.
+        # See https://github.com/rg3/youtube-dl/issues/7202.
+        qs['skip_wall'] = ['1']
+        return compat_urlparse.urlunparse(
+            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
 
 class CrunchyrollIE(CrunchyrollBaseIE):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
-    _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
@@ -81,7 +113,6 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             # rtmp
             'skip_download': True,
         },
-
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -94,24 +125,6 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         '1080': ('80', '108'),
     }
 
-    def _login(self):
-        (username, password) = self._get_login_info()
-        if username is None:
-            return
-        self.report_login()
-        login_url = 'https://www.crunchyroll.com/?a=formhandler'
-        data = urlencode_postdata({
-            'formname': 'RpcApiUser_Login',
-            'name': username,
-            'password': password,
-        })
-        login_request = compat_urllib_request.Request(login_url, data)
-        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        self._download_webpage(login_request, None, False, 'Wrong login info')
-
-    def _real_initialize(self):
-        self._login()
-
     def _decrypt_subtitles(self, data, iv, id):
         data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
         iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
@@ -254,7 +267,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         else:
             webpage_url = 'http://www.' + mobj.group('url')
 
-        webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
+        webpage = self._download_webpage(self._add_skip_wall(webpage_url), video_id, 'Downloading webpage')
         note_m = self._html_search_regex(
             r'<div class="showmedia-trailer-notice">(.+?)</div>',
             webpage, 'trailer-notice', default='')
@@ -352,7 +365,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     IE_NAME = "crunchyroll:playlist"
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?(?:\?|$)'
 
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
@@ -366,7 +379,7 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, show_id)
+        webpage = self._download_webpage(self._add_skip_wall(url), show_id)
         title = self._html_search_regex(
             r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
             webpage, 'title')

From 49941c4e4f6e33785a3be1e0d103bd81657d8a0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 07:06:47 +0600
Subject: [PATCH 1615/2145] [crunchyroll] Add maturity wall reference tests
 (#7202)

---
 youtube_dl/extractor/crunchyroll.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index aa258bbc2..cecd0c784 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -116,6 +116,10 @@ class CrunchyrollIE(CrunchyrollBaseIE):
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
+    }, {
+        # geo-restricted (US), 18+ maturity wall, non-premium available
+        'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
+        'only_matching': True,
     }]
 
     _FORMAT_IDS = {
@@ -374,6 +378,19 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
             'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
         },
         'playlist_count': 13,
+    }, {
+        # geo-restricted (US), 18+ maturity wall, non-premium available
+        'url': 'http://www.crunchyroll.com/cosplay-complex-ova',
+        'info_dict': {
+            'id': 'cosplay-complex-ova',
+            'title': 'Cosplay Complex OVA'
+        },
+        'playlist_count': 3,
+        'skip': 'Georestricted',
+    }, {
+        # geo-restricted (US), 18+ maturity wall, non-premium will be available since 2015.11.14
+        'url': 'http://www.crunchyroll.com/ladies-versus-butlers?skip_wall=1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 448ef1f31c8bcc1550cf907fd46e31026ec981b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:11:02 +0600
Subject: [PATCH 1616/2145] [extractor/common] Allow angle brackets in
 attributes in _og_regexes (#7215)

---
 test/test_InfoExtractor.py     | 4 ++++
 youtube_dl/extractor/common.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 2a00d09a5..938466a80 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -37,12 +37,16 @@ class TestInfoExtractor(unittest.TestCase):
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
             <meta content='application/x-shockwave-flash' property='og:video:type'>
             <meta content='Foo' property=og:foobar>
+            <meta name="og:test1" content='foo > < bar'/>
+            <meta name="og:test2" content="foo >//< bar"/>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
         self.assertEqual(ie._og_search_video_url(html, default=None), None)
         self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
+        self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
+        self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
 
     def test_html_search_meta(self):
         ie = self.ie
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a0c4af92f..4365077f1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -645,7 +645,7 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
+        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
         property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
                        % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'

From 94a773feb94a20be66526348a57ebe20495eba3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 22:25:08 +0200
Subject: [PATCH 1617/2145] [vine] Use JS data to get title/alt_title

---
 youtube_dl/extractor/vine.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c733a48fa..d80b580a0 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -51,6 +51,21 @@ class VineIE(InfoExtractor):
     }, {
         'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
         'only_matching': True,
+    }, {
+        'url': 'https://vine.co/v/e192BnZnZ9V',
+        'info_dict': {
+            'id': 'e192BnZnZ9V',
+            'ext': 'mp4',
+            'title': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
+            'alt_title': 'Vine by Pimry_zaa',
+            'description': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
+            'upload_date': '20150705',
+            'uploader': 'Pimry_zaa',
+            'uploader_id': '1135760698325307392',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -74,8 +89,8 @@ class VineIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'alt_title': self._og_search_description(webpage, default=None),
+            'title': data['description'],
+            'alt_title': 'Vine by %s' % data['username'],
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From 10c38c7ca248d06c2c0f069c5a810e27e207c61e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 22:29:49 +0200
Subject: [PATCH 1618/2145] [vine] Fix download tests

---
 youtube_dl/extractor/vine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d80b580a0..d1dbec893 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -29,10 +29,10 @@ class VineIE(InfoExtractor):
             'id': 'MYxVapFvz2z',
             'ext': 'mp4',
             'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
-            'alt_title': 'Vine by Luna',
+            'alt_title': 'Vine by Mars Ruiz',
             'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
             'upload_date': '20140815',
-            'uploader': 'Luna',
+            'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
         },
     }, {

From 91816e8f16408a3a2753fb254a9e963ad9429ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:32:08 +0600
Subject: [PATCH 1619/2145] [vine] Remove duplicate metadata, make more robust
 and modernize (Closes #7215)

---
 youtube_dl/extractor/vine.py | 39 ++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d1dbec893..6e72cc253 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -1,10 +1,14 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 
 
 class VineIE(InfoExtractor):
@@ -17,7 +21,6 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Chicken.',
             'alt_title': 'Vine by Jack Dorsey',
-            'description': 'Chicken.',
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
@@ -30,7 +33,6 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
             'alt_title': 'Vine by Mars Ruiz',
-            'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
             'upload_date': '20140815',
             'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
@@ -43,7 +45,6 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': '#mw3 #ac130 #killcam #angelofdeath',
             'alt_title': 'Vine by Z3k3',
-            'description': '#mw3 #ac130 #killcam #angelofdeath',
             'upload_date': '20130430',
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
@@ -56,9 +57,8 @@ class VineIE(InfoExtractor):
         'info_dict': {
             'id': 'e192BnZnZ9V',
             'ext': 'mp4',
-            'title': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
+            'title': 'ยิ้ม~ เขิน~ อาย~ น่าร้ากอ้ะ >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
             'alt_title': 'Vine by Pimry_zaa',
-            'description': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
             'upload_date': '20150705',
             'uploader': 'Pimry_zaa',
             'uploader_id': '1135760698325307392',
@@ -80,25 +80,26 @@ class VineIE(InfoExtractor):
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,
-            'vcodec': f['format'],
-            'quality': f['rate'],
+            'vcodec': f.get('format'),
+            'quality': f.get('rate'),
             'url': f['videoUrl'],
-        } for f in data['videoUrls']]
+        } for f in data['videoUrls'] if f.get('videoUrl')]
 
         self._sort_formats(formats)
 
+        username = data.get('username')
+
         return {
             'id': video_id,
-            'title': data['description'],
-            'alt_title': 'Vine by %s' % data['username'],
-            'description': data['description'],
-            'thumbnail': data['thumbnailUrl'],
-            'upload_date': unified_strdate(data['created']),
-            'uploader': data['username'],
-            'uploader_id': data['userIdStr'],
-            'like_count': data['likes']['count'],
-            'comment_count': data['comments']['count'],
-            'repost_count': data['reposts']['count'],
+            'title': data.get('description') or self._og_search_title(webpage),
+            'alt_title': 'Vine by %s' % username if username else self._og_search_description(webpage, default=None),
+            'thumbnail': data.get('thumbnailUrl'),
+            'upload_date': unified_strdate(data.get('created')),
+            'uploader': username,
+            'uploader_id': data.get('userIdStr'),
+            'like_count': int_or_none(data.get('likes', {}).get('count')),
+            'comment_count': int_or_none(data.get('comments', {}).get('count')),
+            'repost_count': int_or_none(data.get('reposts', {}).get('count')),
             'formats': formats,
         }
 

From 02835c6bf4403a907c058d43220a83b3b427e181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:34:54 +0600
Subject: [PATCH 1620/2145] [extractor/common] Document repost_count

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4365077f1..6169fbbeb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -172,6 +172,7 @@ class InfoExtractor(object):
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
+    repost_count:   Number of reposts of the video
     average_rating: Average rating give by users, the scale used depends on the webpage
     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following

From 2e022397c45fbcfd2ef6da43d14b0770221aabd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:36:19 +0600
Subject: [PATCH 1621/2145] [vine] Add counters to tests

---
 youtube_dl/extractor/vine.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 6e72cc253..be72f3147 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -24,6 +24,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
     }, {
         'url': 'https://vine.co/v/MYxVapFvz2z',
@@ -36,6 +39,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20140815',
             'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
     }, {
         'url': 'https://vine.co/v/bxVjBbZlPUH',
@@ -48,6 +54,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20130430',
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
     }, {
         'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
@@ -62,6 +71,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20150705',
             'uploader': 'Pimry_zaa',
             'uploader_id': '1135760698325307392',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
         'params': {
             'skip_download': True,

From 1e399778ee870ee583135e65458268cd7c0fb923 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 20:03:05 +0800
Subject: [PATCH 1622/2145] [letv] Fix extraction

Using data URIs for passing the decrypted M3U8 manifest, which is
supported by ffmpeg only.
---
 youtube_dl/extractor/letv.py | 70 ++++++++++++++++++++++++++----------
 youtube_dl/utils.py          |  5 +++
 2 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index a28abb0f0..9ebbc8089 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -9,13 +9,14 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-    compat_urlparse,
+    compat_ord,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
     int_or_none,
+    encode_data_uri,
 )
 
 
@@ -25,15 +26,16 @@ class LetvIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.letv.com/ptv/vplay/22005890.html',
-        'md5': 'cab23bd68d5a8db9be31c9a222c1e8df',
+        'md5': 'edadcfe5406976f42f9f266057ee5e40',
         'info_dict': {
             'id': '22005890',
             'ext': 'mp4',
             'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
-            'timestamp': 1424747397,
-            'upload_date': '20150224',
             'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
-        }
+        },
+        'params': {
+            'hls_prefer_native': True,
+        },
     }, {
         'url': 'http://www.letv.com/ptv/vplay/1415246.html',
         'info_dict': {
@@ -42,16 +44,22 @@ class LetvIE(InfoExtractor):
             'title': '美人天下01',
             'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
         },
+        'params': {
+            'hls_prefer_native': True,
+        },
     }, {
         'note': 'This video is available only in Mainland China, thus a proxy is needed',
         'url': 'http://www.letv.com/ptv/vplay/1118082.html',
-        'md5': 'f80936fbe20fb2f58648e81386ff7927',
+        'md5': '2424c74948a62e5f31988438979c5ad1',
         'info_dict': {
             'id': '1118082',
             'ext': 'mp4',
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
+        'params': {
+            'hls_prefer_native': True,
+        },
         'skip': 'Only available in China',
     }]
 
@@ -74,6 +82,27 @@ class LetvIE(InfoExtractor):
         _loc3_ = self.ror(_loc3_, _loc2_ % 17)
         return _loc3_
 
+    # see M3U8Encryption class in KLetvPlayer.swf
+    @staticmethod
+    def decrypt_m3u8(encrypted_data):
+        if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
+            return encrypted_data
+        encrypted_data = encrypted_data[5:]
+
+        _loc4_ = bytearray()
+        while encrypted_data:
+            b = compat_ord(encrypted_data[0])
+            _loc4_.extend([b // 16, b & 0x0f])
+            encrypted_data = encrypted_data[1:]
+        idx = len(_loc4_) - 11
+        _loc4_ = _loc4_[idx:] + _loc4_[:idx]
+        _loc7_ = bytearray()
+        while _loc4_:
+            _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
+            _loc4_ = _loc4_[2:]
+
+        return bytes(_loc7_)
+
     def _real_extract(self, url):
         media_id = self._match_id(url)
         page = self._download_webpage(url, media_id)
@@ -115,23 +144,28 @@ class LetvIE(InfoExtractor):
         for format_id in formats:
             if format_id in dispatch:
                 media_url = playurl['domain'][0] + dispatch[format_id][0]
-
-                # Mimic what flvxz.com do
-                url_parts = list(compat_urlparse.urlparse(media_url))
-                qs = dict(compat_urlparse.parse_qs(url_parts[4]))
-                qs.update({
-                    'platid': '14',
-                    'splatid': '1401',
-                    'tss': 'no',
-                    'retry': 1
+                media_url += '&' + compat_urllib_parse.urlencode({
+                    'm3v': 1,
+                    'format': 1,
+                    'expect': 3,
+                    'rateid': format_id,
                 })
-                url_parts[4] = compat_urllib_parse.urlencode(qs)
-                media_url = compat_urlparse.urlunparse(url_parts)
+
+                nodes_data = self._download_json(
+                    media_url, media_id,
+                    'Download JSON metadata for format %s' % format_id)
+
+                req = self._request_webpage(
+                    nodes_data['nodelist'][0]['location'], media_id,
+                    note='Downloading m3u8 information for format %s' % format_id)
+
+                m3u8_data = self.decrypt_m3u8(req.read())
 
                 url_info_dict = {
-                    'url': media_url,
+                    'url': encode_data_uri(m3u8_data, 'application/x-mpegURL'),
                     'ext': determine_ext(dispatch[format_id][1]),
                     'format_id': format_id,
+                    'protocol': 'm3u8',
                 }
 
                 if format_id[-1:] == 'p':
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7dbe25661..db5b3698e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3,6 +3,7 @@
 
 from __future__ import unicode_literals
 
+import base64
 import calendar
 import codecs
 import contextlib
@@ -1795,6 +1796,10 @@ def urlhandle_detect_ext(url_handle):
     return mimetype2ext(getheader('Content-Type'))
 
 
+def encode_data_uri(data, mime_type):
+    return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
+
+
 def age_restricted(content_limit, age_limit):
     """ Returns True iff the content should be blocked """
 

From 985e4fdc07f00a3fdc8e7b7b4119471ee97f3890 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 22:49:05 +0800
Subject: [PATCH 1623/2145] [downloader/hls] Add headers only for http(s) URLs

ffmpeg 2.8.1 raises an error with -headers and non-http input files.
---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index a62d2047b..9a83a73dd 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -30,7 +30,7 @@ class HlsFD(FileDownloader):
 
         args = [ffpp.executable, '-y']
 
-        if info_dict['http_headers']:
+        if info_dict['http_headers'] and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
             args += [

From 0a67a3632bb9cf76f64658986defc1947090ef50 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 23:15:01 +0800
Subject: [PATCH 1624/2145] [compat] Add compat_urllib_request_DataHandler

---
 youtube_dl/compat.py | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 192e1c515..d103ab9ad 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
+import binascii
 import collections
+import email
 import getpass
+import io
 import optparse
 import os
 import re
@@ -38,6 +41,11 @@ try:
 except ImportError:  # Python 2
     import urlparse as compat_urlparse
 
+try:
+    import urllib.response as compat_urllib_response
+except ImportError:  # Python 2
+    import urllib as compat_urllib_response
+
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
@@ -155,6 +163,40 @@ except ImportError:  # Python 2
         string = string.replace('+', ' ')
         return compat_urllib_parse_unquote(string, encoding, errors)
 
+try:
+    from urllib.request import DataHandler as compat_urllib_request_DataHandler
+except ImportError:  # Python < 3.4
+    # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
+    class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
+        def data_open(self, req):
+            # data URLs as specified in RFC 2397.
+            #
+            # ignores POSTed data
+            #
+            # syntax:
+            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
+            # mediatype := [ type "/" subtype ] *( ";" parameter )
+            # data      := *urlchar
+            # parameter := attribute "=" value
+            url = req.get_full_url()
+
+            scheme, data = url.split(":", 1)
+            mediatype, data = data.split(",", 1)
+
+            # even base64 encoded data URLs might be quoted so unquote in any case:
+            data = compat_urllib_parse_unquote_to_bytes(data)
+            if mediatype.endswith(";base64"):
+                data = binascii.a2b_base64(data)
+                mediatype = mediatype[:-7]
+
+            if not mediatype:
+                mediatype = "text/plain;charset=US-ASCII"
+
+            headers = email.message_from_string(
+                "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
+
+            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
+
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
@@ -489,6 +531,8 @@ __all__ = [
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
+    'compat_urllib_request_DataHandler',
+    'compat_urllib_response',
     'compat_urlparse',
     'compat_urlretrieve',
     'compat_xml_parse_error',

From 8b172c2e10fb38c62c213673304c7e8dcd17b768 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 23:16:40 +0800
Subject: [PATCH 1625/2145] [YoutubeDL] Use DataHandler

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index adf70d658..12977bf80 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -37,6 +37,7 @@ from .compat import (
     compat_tokenize_tokenize,
     compat_urllib_error,
     compat_urllib_request,
+    compat_urllib_request_DataHandler,
 )
 from .utils import (
     ContentTooShortError,
@@ -1967,8 +1968,9 @@ class YoutubeDL(object):
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
+        data_handler = compat_urllib_request_DataHandler()
         opener = compat_urllib_request.build_opener(
-            proxy_handler, https_handler, cookie_processor, ydlh)
+            proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

From 48aae2d2cf49843d0efa227fa393a0c783fc3c1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:07:48 +0800
Subject: [PATCH 1626/2145] [twitter] Update tests

---
 youtube_dl/extractor/twitter.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1dd43ff3c..b2fff73b9 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -15,7 +16,7 @@ class TwitterCardIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-            'md5': 'a74f50b310c83170319ba16de6955192',
+            'md5': '7d2f6b4d2eb841a7ccc893d479bfceb4',
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
@@ -103,17 +104,17 @@ class TwitterIE(TwitterCardIE):
     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 
     _TESTS = [{
-        'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
-        'md5': '8bbccb487bd7a31349b775915fcd412f',
+        'url': 'https://twitter.com/freethenipple/status/643211948184596480',
+        'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
         'info_dict': {
-            'id': '614301758345490432',
+            'id': '643211948184596480',
             'ext': 'mp4',
-            'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
+            'title': 'freethenipple - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
-            'duration': 29.5,
-            'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
-            'uploader': 'banksy',
-            'uploader_id': 'thereaIbanksy',
+            'duration': 12.922,
+            'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
+            'uploader': 'FREE THE NIPPLE',
+            'uploader_id': 'freethenipple',
         },
     }]
 

From 01d22d47039dedace1c5414c83e9fecfca41b5a5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:11:55 +0800
Subject: [PATCH 1627/2145] [twitter] Use _download_xml

---
 youtube_dl/extractor/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index b2fff73b9..37a9fd5fd 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -8,6 +8,7 @@ from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     unescapeHTML,
+    xpath_text,
 )
 
 
@@ -60,9 +61,8 @@ class TwitterCardIE(InfoExtractor):
                 video_id)
             if 'playlist' not in config:
                 if 'vmapUrl' in config:
-                    webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
-                    video_url = self._search_regex(
-                        r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
+                    vmap_data = self._download_xml(config['vmapUrl'], video_id)
+                    video_url = xpath_text(vmap_data, './/MediaFile').strip()
                     f = {
                         'url': video_url,
                     }

From 014e880372e896cdd63f9075864d2a3bba60e706 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:13:58 +0800
Subject: [PATCH 1628/2145] [twitter] Add IE_NAMEs

---
 youtube_dl/extractor/twitter.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 37a9fd5fd..5f697782e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class TwitterCardIE(InfoExtractor):
+    IE_NAME = 'twitter:card'
     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
     _TESTS = [
         {
@@ -101,6 +102,7 @@ class TwitterCardIE(InfoExtractor):
 
 
 class TwitterIE(TwitterCardIE):
+    IE_NAME = 'twitter'
     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 
     _TESTS = [{

From f322bfb0638aeeb527459ebcf00f8a3dde26280c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:15:47 +0800
Subject: [PATCH 1629/2145] [twitter:card] Remove unneeded 'ext'

---
 youtube_dl/extractor/twitter.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5f697782e..48bef5d80 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -64,13 +64,9 @@ class TwitterCardIE(InfoExtractor):
                 if 'vmapUrl' in config:
                     vmap_data = self._download_xml(config['vmapUrl'], video_id)
                     video_url = xpath_text(vmap_data, './/MediaFile').strip()
-                    f = {
+                    formats.append({
                         'url': video_url,
-                    }
-                    ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
-                    if ext:
-                        f['ext'] = ext.group(1)
-                    formats.append(f)
+                    })
                     break   # same video regardless of UA
                 continue
 

From e04edad621efe56347e155b6dc59a0c3d589b3bd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:16:57 +0800
Subject: [PATCH 1630/2145] [twitter] Inherit from InfoExtractor directly

---
 youtube_dl/extractor/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 48bef5d80..c9b783745 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -97,11 +97,11 @@ class TwitterCardIE(InfoExtractor):
         }
 
 
-class TwitterIE(TwitterCardIE):
+class TwitterIE(InfoExtractor):
     IE_NAME = 'twitter'
     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 
-    _TESTS = [{
+    _TEST = {
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
         'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
         'info_dict': {
@@ -114,7 +114,7 @@ class TwitterIE(TwitterCardIE):
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
         },
-    }]
+    }
 
     def _real_extract(self, url):
         id = self._match_id(url)

From f6dfd6603a9e9bb88ebcdcd52490974a34d1bd11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:18:01 +0800
Subject: [PATCH 1631/2145] [twitter] Use _html_search_regex

---
 youtube_dl/extractor/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index c9b783745..6ff15369c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -122,7 +122,7 @@ class TwitterIE(InfoExtractor):
         name = username
         url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
         webpage = self._download_webpage(url, 'tweet: ' + url)
-        description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
+        description = self._html_search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title')
         title = description.replace('\n', ' ')
         splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
         if splitdesc:

From 575036b40504bc921b18f05bde64e0e7dceacec6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:04:13 +0800
Subject: [PATCH 1632/2145] [twitter] Simplify and improve

---
 youtube_dl/extractor/twitter.py | 41 +++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6ff15369c..6b3b39aee 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     unescapeHTML,
     xpath_text,
+    remove_end,
 )
 
 
@@ -99,7 +100,8 @@ class TwitterCardIE(InfoExtractor):
 
 class TwitterIE(InfoExtractor):
     IE_NAME = 'twitter'
-    _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
+    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
+    _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
 
     _TEST = {
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -107,7 +109,7 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',
-            'title': 'freethenipple - FTN supporters on Hollywood Blvd today!',
+            'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 12.922,
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
@@ -117,26 +119,31 @@ class TwitterIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        id = self._match_id(url)
-        username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
-        name = username
-        url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
-        webpage = self._download_webpage(url, 'tweet: ' + url)
-        description = self._html_search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title')
-        title = description.replace('\n', ' ')
-        splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
-        if splitdesc:
-            name, title = splitdesc.groups()
-        title = re.sub(r'\s*https?://[^ ]+', '', title)  # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
-        card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
+        mobj = re.match(self._VALID_URL, url)
+        user_id = mobj.group('user_id')
+        twid = mobj.group('id')
+
+        webpage = self._download_webpage(self._TEMPLATE_URL % (user_id, twid), twid)
+
+        username = remove_end(self._og_search_title(webpage), ' on Twitter')
+
+        title = self._og_search_description(webpage).strip('').replace('\n', ' ')
+
+        # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
+        mobj = re.match(r'“(.*)\s+(http://[^ ]+)”', title)
+        title, short_url = mobj.groups()
+
+        card_id = self._search_regex(
+            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url')
         card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+
         return {
             '_type': 'url_transparent',
             'ie_key': 'TwitterCard',
-            'uploader_id': username,
-            'uploader': name,
+            'uploader_id': user_id,
+            'uploader': username,
             'url': card_url,
             'webpage_url': url,
-            'description': description,
+            'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
             'title': username + ' - ' + title,
         }

From 77a54b6a658059a11de415d793588fdbfec14194 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:08:24 +0800
Subject: [PATCH 1633/2145] [twitter:card] Use _html_search_regex

---
 youtube_dl/extractor/twitter.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6b3b39aee..1cdca544c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
-    unescapeHTML,
     xpath_text,
     remove_end,
 )
@@ -57,9 +56,8 @@ class TwitterCardIE(InfoExtractor):
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
-            config = self._parse_json(
-                unescapeHTML(self._search_regex(
-                    r'data-player-config="([^"]+)"', webpage, 'data player config')),
+            config = self._parse_json(self._html_search_regex(
+                r'data-player-config="([^"]+)"', webpage, 'data player config'),
                 video_id)
             if 'playlist' not in config:
                 if 'vmapUrl' in config:

From c88aec845a680ef9404b637b3dbcf706dcf00b68 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:23:56 +0800
Subject: [PATCH 1634/2145] [twitter] Fix short URL extraction

---
 youtube_dl/extractor/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1cdca544c..1472f22a7 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -128,7 +128,7 @@ class TwitterIE(InfoExtractor):
         title = self._og_search_description(webpage).strip('').replace('\n', ' ')
 
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
-        mobj = re.match(r'“(.*)\s+(http://[^ ]+)”', title)
+        mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
         title, short_url = mobj.groups()
 
         card_id = self._search_regex(

From 4a7b79038425f614af49116edab7897f0db13e5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 19:07:37 +0800
Subject: [PATCH 1635/2145] [twitter:card] Support YouTube embeds

---
 youtube_dl/extractor/twitter.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1472f22a7..9d3e46b94 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -37,6 +37,19 @@ class TwitterCardIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 80.155,
             },
+        },
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
+            'md5': 'b6f35e8b08a0bec6c8af77a2f4b3a814',
+            'info_dict': {
+                'id': 'dq4Oj5quskI',
+                'ext': 'mp4',
+                'title': 'Ubuntu 11.10 Overview',
+                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10-things-to-do-after-installing-ubuntu-11-10/',
+                'upload_date': '20111013',
+                'uploader': 'OMG! Ubuntu!',
+                'uploader_id': 'omgubuntu',
+            },
         }
     ]
 
@@ -56,6 +69,12 @@ class TwitterCardIE(InfoExtractor):
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
+            youtube_url = self._html_search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+                webpage, 'youtube iframe', default=None)
+            if youtube_url:
+                return self.url_result(youtube_url, 'Youtube')
+
             config = self._parse_json(self._html_search_regex(
                 r'data-player-config="([^"]+)"', webpage, 'data player config'),
                 video_id)

From 05a3879f1c142cc2bf0287cde4690d8ccadcdc8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 19:19:46 +0800
Subject: [PATCH 1636/2145] [letv] Update M3U8's MIME type

The new MIME type appears in the following places:
https://www.iana.org/assignments/media-types/media-types.xhtml#application
https://hg.python.org/cpython/file/tip/Lib/mimetypes.py
---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9ebbc8089..effd9eb92 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -162,7 +162,7 @@ class LetvIE(InfoExtractor):
                 m3u8_data = self.decrypt_m3u8(req.read())
 
                 url_info_dict = {
-                    'url': encode_data_uri(m3u8_data, 'application/x-mpegURL'),
+                    'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
                     'ext': determine_ext(dispatch[format_id][1]),
                     'format_id': format_id,
                     'protocol': 'm3u8',

From dd67702a3ea007369109ee8e4b67043064e1f759 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 18 Oct 2015 14:13:06 +0200
Subject: [PATCH 1637/2145] [imdb] Fix extraction (fixes #7220)

---
 youtube_dl/extractor/imdb.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 4bb574cf3..02e1e428e 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -4,8 +4,8 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
+from ..utils import (
+    qualities,
 )
 
 
@@ -30,24 +30,33 @@ class ImdbIE(InfoExtractor):
         descr = self._html_search_regex(
             r'(?s)<span itemprop="description">(.*?)</span>',
             webpage, 'description', fatal=False)
-        available_formats = re.findall(
-            r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,
-            flags=re.MULTILINE)
+        player_url = 'http://www.imdb.com/video/imdb/vi%s/imdb/single' % video_id
+        player_page = self._download_webpage(
+            player_url, video_id, 'Downloading player page')
+        # the player page contains the info for the default format, we have to
+        # fetch other pages for the rest of the formats
+        extra_formats = re.findall(r'href="(?P<url>%s.*?)".*?>(?P<name>.*?)<' % re.escape(player_url), player_page)
+        format_pages = [
+            self._download_webpage(
+                f_url, video_id, 'Downloading info for %s format' % f_name)
+            for f_url, f_name in extra_formats]
+        format_pages.append(player_page)
+
+        quality = qualities(['SD', '480p', '720p'])
         formats = []
-        for f_id, f_path in available_formats:
-            f_path = f_path.strip()
-            format_page = self._download_webpage(
-                compat_urlparse.urljoin(url, f_path),
-                'Downloading info for %s format' % f_id)
+        for format_page in format_pages:
             json_data = self._search_regex(
                 r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
                 format_page, 'json data', flags=re.DOTALL)
             info = json.loads(json_data)
             format_info = info['videoPlayerObject']['video']
+            f_id = format_info['ffname']
             formats.append({
                 'format_id': f_id,
                 'url': format_info['videoInfoList'][0]['videoUrl'],
+                'quality': quality(f_id),
             })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From b0f001a6cbd220c8b10c0ce359f17072d6347a8f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Sep 2015 15:52:36 +0100
Subject: [PATCH 1638/2145] [canalc2] fix info extraction

---
 youtube_dl/extractor/canalc2.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index c4fefefe4..66a9ff093 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -8,34 +8,40 @@ from .common import InfoExtractor
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?.*?idVideo=(?P<id>\d+)'
+    _VALID_URL = r'https?://(www\.)?canalc2\.tv/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.canalc2.tv/video.asp?idVideo=12163&voir=oui',
+        'url': 'http://www.canalc2.tv/video/12163',
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
             'id': '12163',
             'ext': 'mp4',
             'title': 'Terrasses du Numérique'
+        },
+        'params': {
+            'skip_download': True,  # Requires rtmpdump
         }
     }
 
     def _real_extract(self, url):
-        video_id = re.match(self._VALID_URL, url).group('id')
-        # We need to set the voir field for getting the file name
-        url = 'http://www.canalc2.tv/video.asp?idVideo=%s&voir=oui' % video_id
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        file_name = self._search_regex(
-            r"so\.addVariable\('file','(.*?)'\);",
-            webpage, 'file name')
-        video_url = 'http://vod-flash.u-strasbg.fr:8080/' + file_name
+        video_url = self._search_regex(
+            r'jwplayer\("Player"\).setup\({[^}]*file: "([^"]+)"',
+            webpage, 'video_url')
+        formats = [{'url': video_url}]
+        if video_url.startswith('rtmp://'):
+            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<play_path>mp4:.+)$', video_url)
+            formats[0].update({
+                'app': rtmp.group('app'),
+                'play_path': rtmp.group('play_path'),
+            })
 
         title = self._html_search_regex(
-            r'class="evenement8">(.*?)</a>', webpage, 'title')
+            r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
 
         return {
             'id': video_id,
-            'ext': 'mp4',
-            'url': video_url,
+            'formats': formats,
             'title': title,
         }

From 6682049dee5e73b98e99e1359b959240d0920d6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:19:43 +0600
Subject: [PATCH 1639/2145] [canalc2] Improve rtmp extraction

---
 youtube_dl/extractor/canalc2.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 66a9ff093..648af2e18 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -31,10 +31,12 @@ class Canalc2IE(InfoExtractor):
             webpage, 'video_url')
         formats = [{'url': video_url}]
         if video_url.startswith('rtmp://'):
-            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<play_path>mp4:.+)$', video_url)
+            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
             formats[0].update({
+                'url': rtmp.group('url'),
                 'app': rtmp.group('app'),
                 'play_path': rtmp.group('play_path'),
+                'page_url': url,
             })
 
         title = self._html_search_regex(

From ef6c868f23f2fe0d493831e0d4cba71c735bd160 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:23:31 +0600
Subject: [PATCH 1640/2145] [canalc2] Improve some regexes

---
 youtube_dl/extractor/canalc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 648af2e18..d9137e2ef 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'https?://(www\.)?canalc2\.tv/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?canalc2\.tv/video/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.canalc2.tv/video/12163',
@@ -27,8 +27,8 @@ class Canalc2IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r'jwplayer\("Player"\).setup\({[^}]*file: "([^"]+)"',
-            webpage, 'video_url')
+            r'jwplayer\((["\'])Player\1\)\.setup\({[^}]*file\s*:\s*(["\'])(?P<file>.+?)\2',
+            webpage, 'video_url', group='file')
         formats = [{'url': video_url}]
         if video_url.startswith('rtmp://'):
             rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)

From 14bddf35fbe8253e283042630e24b134996b2575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:23:52 +0600
Subject: [PATCH 1641/2145] [canalc2] Add ext

---
 youtube_dl/extractor/canalc2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index d9137e2ef..ba82bb2b7 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -34,6 +34,7 @@ class Canalc2IE(InfoExtractor):
             rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
             formats[0].update({
                 'url': rtmp.group('url'),
+                'ext': 'flv',
                 'app': rtmp.group('app'),
                 'play_path': rtmp.group('play_path'),
                 'page_url': url,

From b1bf063503893192637f95e929d1a9147de59a7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:27:05 +0600
Subject: [PATCH 1642/2145] [canalc2] Extract duration

---
 youtube_dl/extractor/canalc2.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index ba82bb2b7..e326b8fbd 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_duration
 
 
 class Canalc2IE(InfoExtractor):
@@ -42,9 +43,13 @@ class Canalc2IE(InfoExtractor):
 
         title = self._html_search_regex(
             r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
+        duration = parse_duration(self._search_regex(
+            r'id=["\']video_duree["\'][^>]*>([^<]+)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
-            'formats': formats,
             'title': title,
+            'duration': duration,
+            'formats': formats,
         }

From 608945d44a7e47fa5115295839c993af545936eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:27:22 +0600
Subject: [PATCH 1643/2145] [canalc2] Fix test

---
 youtube_dl/extractor/canalc2.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index e326b8fbd..f6a1ff381 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -16,8 +16,9 @@ class Canalc2IE(InfoExtractor):
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
             'id': '12163',
-            'ext': 'mp4',
-            'title': 'Terrasses du Numérique'
+            'ext': 'flv',
+            'title': 'Terrasses du Numérique',
+            'duration': 122,
         },
         'params': {
             'skip_download': True,  # Requires rtmpdump

From dedd35c6bc33eb88f19b16eeb37498cee076c47a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:59:18 +0600
Subject: [PATCH 1644/2145] [viewster] Fix failing m3u8

---
 youtube_dl/extractor/viewster.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 632e57fb4..7cf930d69 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -131,10 +131,11 @@ class ViewsterIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(
                     video_url, video_id, f4m_id='hds'))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     video_url, video_id, 'mp4', m3u8_id='hls',
-                    fatal=False  # m3u8 sometimes fail
-                ))
+                    fatal=False)  # m3u8 sometimes fail
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
                 format_id = media.get('Bitrate')
                 f = {

From e36963e0eb57294f156a98c38df891dec41ebaa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 20:24:33 +0600
Subject: [PATCH 1645/2145] [eagleplatform] Identify hls formats

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index e529b9b96..7bbf617d4 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -87,7 +87,7 @@ class EaglePlatformIE(InfoExtractor):
         m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
         formats = self._extract_m3u8_formats(
             m3u8_url, video_id,
-            'mp4', entry_protocol='m3u8_native')
+            'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
 
         mp4_url = self._get_video_url(
             # Secure mp4 URL is constructed according to Player.prototype.mp4 from

From a6e0afa2bbc93d145b31911b8ce40c502994e2a1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 18 Oct 2015 19:23:40 +0200
Subject: [PATCH 1646/2145] release 2015.10.18

---
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 47f7da86d..cfa665d88 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -588,7 +588,8 @@
  - **twitch:stream**
  - **twitch:video**
  - **twitch:vod**
- - **TwitterCard**
+ - **twitter**
+ - **twitter:card**
  - **Ubu**
  - **udemy**
  - **udemy:course**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 31d2a9dc0..660b0050b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.16'
+__version__ = '2015.10.18'

From 264b23e1a42378d52f8774a07c1d906cd1cff96c Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 19:56:22 +0200
Subject: [PATCH 1647/2145] adds thumbnail support for ZDF Mediathek extractor

---
 youtube_dl/extractor/zdf.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 98f15177b..f376025e1 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -70,6 +70,23 @@ def extract_from_xml_url(ie, video_id, xml_url):
             '_available': is_available,
         }
 
+    def xml_to_thumbnails(fnode):
+        thumbnails = list()
+        for node in fnode:
+            width_x_height = node.attrib['key']
+            thumbnail = {
+                'url': node.text,
+                'width': int(width_x_height.split('x')[0]),
+                'height': int(width_x_height.split('x')[1])
+            }
+            thumbnails.append(thumbnail)
+        return thumbnails
+
+
+    thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
+    thumbnails = xml_to_thumbnails(thumbnail_nodes)
+    thumbnail = thumbnails[-1]['url']
+
     format_nodes = doc.findall('.//formitaeten/formitaet')
     formats = list(filter(
         lambda f: f['_available'],
@@ -81,6 +98,8 @@ def extract_from_xml_url(ie, video_id, xml_url):
         'title': title,
         'description': description,
         'duration': duration,
+        'thumbnail': thumbnail,
+        'thumbnails': thumbnails,
         'uploader': uploader,
         'uploader_id': uploader_id,
         'upload_date': upload_date,

From d762f86e940ad656e8f7e7b93636292e4cf36de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 00:11:16 +0600
Subject: [PATCH 1648/2145] [ok] Extend _VALID_URL

---
 youtube_dl/extractor/odnoklassniki.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index ccc88cfb1..184c7a323 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
@@ -66,6 +66,9 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://www.ok.ru/video/20648036891',
         'only_matching': True,
+    }, {
+        'url': 'http://www.ok.ru/videoembed/20648036891',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 8cc83d301dd0e8029aff804e362860d36e3d7e7a Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 20:47:42 +0200
Subject: [PATCH 1649/2145] use int_or_none, check if attrib exists, remove
 thumbnail

---
 youtube_dl/extractor/zdf.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index f376025e1..d41c4e712 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -73,19 +73,17 @@ def extract_from_xml_url(ie, video_id, xml_url):
     def xml_to_thumbnails(fnode):
         thumbnails = list()
         for node in fnode:
-            width_x_height = node.attrib['key']
-            thumbnail = {
-                'url': node.text,
-                'width': int(width_x_height.split('x')[0]),
-                'height': int(width_x_height.split('x')[1])
-            }
+            thumbnail = {'url': node.text}
+            if 'key' in node.attrib:
+                width_x_height = node.attrib['key']
+                thumbnail['width'] = int_or_none(width_x_height.split('x')[0])
+                thumbnail['height'] = int_or_none(width_x_height.split('x')[1])
             thumbnails.append(thumbnail)
         return thumbnails
 
 
     thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
     thumbnails = xml_to_thumbnails(thumbnail_nodes)
-    thumbnail = thumbnails[-1]['url']
 
     format_nodes = doc.findall('.//formitaeten/formitaet')
     formats = list(filter(
@@ -98,7 +96,6 @@ def extract_from_xml_url(ie, video_id, xml_url):
         'title': title,
         'description': description,
         'duration': duration,
-        'thumbnail': thumbnail,
         'thumbnails': thumbnails,
         'uploader': uploader,
         'uploader_id': uploader_id,

From b243340f0ce311443a15a2dfd4356a9504e18c04 Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 21:07:52 +0200
Subject: [PATCH 1650/2145] check if key attrib matches resolution pattern

---
 youtube_dl/extractor/zdf.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index d41c4e712..ed385450c 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -75,9 +75,9 @@ def extract_from_xml_url(ie, video_id, xml_url):
         for node in fnode:
             thumbnail = {'url': node.text}
             if 'key' in node.attrib:
-                width_x_height = node.attrib['key']
-                thumbnail['width'] = int_or_none(width_x_height.split('x')[0])
-                thumbnail['height'] = int_or_none(width_x_height.split('x')[1])
+                if re.match("^[0-9]+x[0-9]+$", node.attrib['key']):
+                    thumbnail['width'] = int_or_none(node.attrib['key'].split('x')[0])
+                    thumbnail['height'] = int_or_none(node.attrib['key'].split('x')[1])
             thumbnails.append(thumbnail)
         return thumbnails
 

From 2038ad6ee71c842420b83cb6c5ce3c6898e8e380 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 01:12:41 +0600
Subject: [PATCH 1651/2145] [README.md] Add uploader extraction sample in
 example extractor

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cf4aebf3d..a6ec9619c 100644
--- a/README.md
+++ b/README.md
@@ -710,12 +710,13 @@ If you want to add support for a new site, you can follow this quick list (assum
             webpage = self._download_webpage(url, video_id)
 
             # TODO more code goes here, for example ...
-            title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+            title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
 
             return {
                 'id': video_id,
                 'title': title,
                 'description': self._og_search_description(webpage),
+                'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```

From b7cedb16043c60d4032b206a83539acbd39f994f Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 21:25:26 +0200
Subject: [PATCH 1652/2145] simplify thumbnail dict building

---
 youtube_dl/extractor/zdf.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index ed385450c..c2b196504 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -75,9 +75,10 @@ def extract_from_xml_url(ie, video_id, xml_url):
         for node in fnode:
             thumbnail = {'url': node.text}
             if 'key' in node.attrib:
-                if re.match("^[0-9]+x[0-9]+$", node.attrib['key']):
-                    thumbnail['width'] = int_or_none(node.attrib['key'].split('x')[0])
-                    thumbnail['height'] = int_or_none(node.attrib['key'].split('x')[1])
+                m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
+                if m:
+                    thumbnail['width'] = int(m.group(1))
+                    thumbnail['height'] = int(m.group(2))
             thumbnails.append(thumbnail)
         return thumbnails
 

From 7b091c370c0f187545df8b1b1cc990fcf95df108 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 01:48:05 +0600
Subject: [PATCH 1653/2145] [zdf] Modernize and PEP 8

---
 youtube_dl/extractor/zdf.py | 43 +++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index c2b196504..a795f56b3 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -9,6 +9,7 @@ from ..utils import (
     int_or_none,
     unified_strdate,
     OnDemandPagedList,
+    xpath_text,
 )
 
 
@@ -19,13 +20,11 @@ def extract_from_xml_url(ie, video_id, xml_url):
         errnote='Failed to download video info')
 
     title = doc.find('.//information/title').text
-    description = doc.find('.//information/detail').text
-    duration = int(doc.find('.//details/lengthSec').text)
-    uploader_node = doc.find('.//details/originChannelTitle')
-    uploader = None if uploader_node is None else uploader_node.text
-    uploader_id_node = doc.find('.//details/originChannelId')
-    uploader_id = None if uploader_id_node is None else uploader_id_node.text
-    upload_date = unified_strdate(doc.find('.//details/airtime').text)
+    description = xpath_text(doc, './/information/detail', 'description')
+    duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
+    uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
+    uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
+    upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
 
     def xml_to_format(fnode):
         video_url = fnode.find('url').text
@@ -40,15 +39,14 @@ def extract_from_xml_url(ie, video_id, xml_url):
         ext = format_m.group('container')
         proto = format_m.group('proto').lower()
 
-        quality = fnode.find('./quality').text
-        abr = int(fnode.find('./audioBitrate').text) // 1000
-        vbr_node = fnode.find('./videoBitrate')
-        vbr = None if vbr_node is None else int(vbr_node.text) // 1000
+        quality = xpath_text(fnode, './quality', 'quality')
+        abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
+        vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
 
-        width_node = fnode.find('./width')
-        width = None if width_node is None else int_or_none(width_node.text)
-        height_node = fnode.find('./height')
-        height = None if height_node is None else int_or_none(height_node.text)
+        width = int_or_none(xpath_text(fnode, './width', 'width'))
+        height = int_or_none(xpath_text(fnode, './height', 'height'))
+
+        filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
 
         format_note = ''
         if not format_note:
@@ -64,16 +62,21 @@ def extract_from_xml_url(ie, video_id, xml_url):
             'vbr': vbr,
             'width': width,
             'height': height,
-            'filesize': int_or_none(fnode.find('./filesize').text),
+            'filesize': filesize,
             'format_note': format_note,
             'protocol': proto,
             '_available': is_available,
         }
 
     def xml_to_thumbnails(fnode):
-        thumbnails = list()
+        thumbnails = []
         for node in fnode:
-            thumbnail = {'url': node.text}
+            thumbnail_url = node.text
+            if not thumbnail_url:
+                continue
+            thumbnail = {
+                'url': thumbnail_url,
+            }
             if 'key' in node.attrib:
                 m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
                 if m:
@@ -82,9 +85,7 @@ def extract_from_xml_url(ie, video_id, xml_url):
             thumbnails.append(thumbnail)
         return thumbnails
 
-
-    thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
-    thumbnails = xml_to_thumbnails(thumbnail_nodes)
+    thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
 
     format_nodes = doc.findall('.//formitaeten/formitaet')
     formats = list(filter(

From 0be30bafa42dbfa99644a9eb7fefa5cebb70f121 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Mon, 19 Oct 2015 20:53:27 +0200
Subject: [PATCH 1654/2145] [vidme] Stream URL fallback, better error message
 for suspended videos

---
 youtube_dl/extractor/vidme.py | 37 +++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 382517a4a..393970a12 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -97,6 +97,31 @@ class VidmeIE(InfoExtractor):
         # nsfw, user-disabled
         'url': 'https://vid.me/dzGJ',
         'only_matching': True,
+    }, {
+        # suspended
+        'url': 'https://vid.me/Ox3G',
+        'only_matching': True,
+    }, {
+        # no formats in the API response
+        'url': 'https://vid.me/e5g',
+        'info_dict': {
+            'id': 'e5g',
+            'ext': 'mp4',
+            'title': 'e5g',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1401480195,
+            'upload_date': '20140530',
+            'uploader': None,
+            'uploader_id': None,
+            'age_limit': 0,
+            'duration': 483,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -118,7 +143,7 @@ class VidmeIE(InfoExtractor):
 
         video = response['video']
 
-        if video.get('state') == 'user-disabled':
+        if video.get('state') in ('user-disabled', 'suspended'):
             raise ExtractorError(
                 'Vidme said: This video has been suspended either due to a copyright claim, '
                 'or for violating the terms of use.',
@@ -131,6 +156,14 @@ class VidmeIE(InfoExtractor):
             'height': int_or_none(f.get('height')),
             'preference': 0 if f.get('type', '').endswith('clip') else 1,
         } for f in video.get('formats', []) if f.get('uri')]
+
+        if not formats and video.get('complete_url'):
+            formats.append({
+                'url': video.get('complete_url'),
+                'width': int_or_none(video.get('width')),
+                'height': int_or_none(video.get('height')),
+            })
+
         self._sort_formats(formats)
 
         title = video['title']
@@ -147,7 +180,7 @@ class VidmeIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': title,
+            'title': title or video_id,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,

From 4bf56141950f3c24000381403417d20095f04460 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 20 Oct 2015 07:43:39 +0100
Subject: [PATCH 1655/2145] [cspan] move get_text_attr to CSpanIE

---
 youtube_dl/extractor/cspan.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index c74b35fd9..388460a32 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -14,10 +14,6 @@ from ..utils import (
 from .senateisvp import SenateISVPIE
 
 
-def get_text_attr(d, attr):
-    return d.get(attr, {}).get('#text')
-
-
 class CSpanIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
     IE_DESC = 'C-SPAN'
@@ -60,6 +56,9 @@ class CSpanIE(InfoExtractor):
         }
     }]
 
+    def get_text_attr(self, d, attr):
+        return d.get(attr, {}).get('#text')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -79,7 +78,7 @@ class CSpanIE(InfoExtractor):
             'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
             video_id)['video']
         if data['@status'] != 'Success':
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, self.get_text_attr(data, 'error')), expected=True)
 
         doc = self._download_xml(
             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -91,17 +90,17 @@ class CSpanIE(InfoExtractor):
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
         files = data['files']
-        capfile = get_text_attr(data, 'capfile')
+        capfile = self.get_text_attr(data, 'capfile')
 
         entries = []
         for partnum, f in enumerate(files):
             formats = []
             for quality in f['qualities']:
                 formats.append({
-                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
-                    'url': unescapeHTML(get_text_attr(quality, 'file')),
-                    'height': int_or_none(get_text_attr(quality, 'height')),
-                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+                    'format_id': '%s-%sp' % (self.get_text_attr(quality, 'bitrate'), self.get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(self.get_text_attr(quality, 'file')),
+                    'height': int_or_none(self.get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(self.get_text_attr(quality, 'bitrate')),
                 })
             self._sort_formats(formats)
             entries.append({
@@ -112,7 +111,7 @@ class CSpanIE(InfoExtractor):
                 'formats': formats,
                 'description': description,
                 'thumbnail': thumbnail,
-                'duration': int_or_none(get_text_attr(f, 'length')),
+                'duration': int_or_none(self.get_text_attr(f, 'length')),
                 'subtitles': {
                     'en': [{
                         'url': capfile,

From b6aa99aff8278142fed94e37e500f1cfb62defd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Tue, 20 Oct 2015 10:30:31 +0200
Subject: [PATCH 1656/2145] [vimeo] Fix error parsing

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0f84656c0..bdec79341 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -273,13 +273,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
         self.report_extraction(video_id)
 
         vimeo_config = self._search_regex(
-            r'vimeo\.config\s*=\s*({.+?});', webpage,
+            r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));', webpage,
             'vimeo config', default=None)
         if vimeo_config:
             seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
             if seed_status.get('state') == 'failed':
                 raise ExtractorError(
-                    '%s returned error: %s' % (self.IE_NAME, seed_status['title']),
+                    '%s said: %s' % (self.IE_NAME, seed_status['title']),
                     expected=True)
 
         # Extract the config JSON

From 4a8963770e37568c484841338cbb6761cf3cb5c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 20:17:54 +0600
Subject: [PATCH 1657/2145] [vidme] Use original vid.me title template for
 untitled videos

---
 youtube_dl/extractor/vidme.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 393970a12..296e00423 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -107,7 +107,7 @@ class VidmeIE(InfoExtractor):
         'info_dict': {
             'id': 'e5g',
             'ext': 'mp4',
-            'title': 'e5g',
+            'title': 'Video upload (e5g)',
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1401480195,
             'upload_date': '20140530',
@@ -180,7 +180,7 @@ class VidmeIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': title or video_id,
+            'title': title or 'Video upload (%s)' % video_id,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,

From d65889bbc0a6b4a1eafe6a8c0e0e26170dc75586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 20:18:23 +0600
Subject: [PATCH 1658/2145] [vidme] Update test

---
 youtube_dl/extractor/vidme.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 296e00423..eb5cde761 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -14,7 +14,7 @@ class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
-        'md5': 'c62f1156138dc3323902188c5b5a8bd6',
+        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
         'info_dict': {
             'id': 'QNB',
             'ext': 'mp4',

From 8bea039b8329074af9a95fe51e7622c8074f6218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Tue, 20 Oct 2015 16:38:44 +0200
Subject: [PATCH 1659/2145] [vimeo] New test, fixed one older test

---
 youtube_dl/extractor/vimeo.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bdec79341..2437ae1eb 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -133,7 +133,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
-                'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people who love them.',
+                'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people\u2026',
             },
             'params': {
                 'videopassword': 'youtube-dl',
@@ -181,6 +181,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user28849593',
             },
         },
+        {
+            'url': 'https://vimeo.com/109815029',
+            'note': 'Video not completely processed, "failed" seed status',
+            'only_matching': True,
+        },
     ]
 
     @staticmethod

From d01949dc89feb2441f251e42e8a6bfa4711b9715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 23:09:51 +0600
Subject: [PATCH 1660/2145] [utils:js_to_json] Fix bad escape in double quoted
 strings

---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a5f164c49..918a7a9ef 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -495,6 +495,9 @@ class TestUtil(unittest.TestCase):
             "playlist":[{"controls":{"all":null}}]
         }''')
 
+        inp = '''"The CW\\'s \\'Crazy Ex-Girlfriend\\'"'''
+        self.assertEqual(js_to_json(inp), '''"The CW's 'Crazy Ex-Girlfriend'"''')
+
         inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
         json_code = js_to_json(inp)
         self.assertEqual(json.loads(json_code), json.loads(inp))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index db5b3698e..a61e47646 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1701,8 +1701,8 @@ def js_to_json(code):
         if v in ('true', 'false', 'null'):
             return v
         if v.startswith('"'):
-            return v
-        if v.startswith("'"):
+            v = re.sub(r"\\'", "'", v[1:-1])
+        elif v.startswith("'"):
             v = v[1:-1]
             v = re.sub(r"\\\\|\\'|\"", lambda m: {
                 '\\\\': '\\\\',

From 4211c83aa4dec0cf9874a6a485665360570e2a89 Mon Sep 17 00:00:00 2001
From: mjdubell <localhost@localhost>
Date: Mon, 19 Oct 2015 03:36:07 +0200
Subject: [PATCH 1661/2145] [stitcher] Add extractor

Stitcher review updates

Removed re import

Stitcher review updates
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/stitcher.py | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/stitcher.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd6eb6ae0..eac5e7d5e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -586,6 +586,7 @@ from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
+from .stitcher import StitcherIE
 from .sport5 import Sport5IE
 from .sportbox import (
     SportBoxIE,
diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
new file mode 100644
index 000000000..a547debbd
--- /dev/null
+++ b/youtube_dl/extractor/stitcher.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class StitcherIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/[\/a-z\-]+(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
+        'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
+        'info_dict': {
+            'id': '40789481',
+            'ext': 'mp3',
+            'title': 'Machine Learning Mastery and Cancer Clusters from Talking Machines',
+        }
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        title = self._og_search_title(webpage)
+        url = self._search_regex(r'episodeURL: "(.+?)"', webpage, 'url')
+        episode_image = self._search_regex(r'episodeImage: "(.+?)"', webpage, 'episode_image', fatal=False)
+        duration = int_or_none(self._search_regex(r'duration: (\d+?),', webpage, 'duration', fatal=False))
+
+        return {
+            'id': audio_id,
+            'url': url,
+            'title': title,
+            'duration': duration,
+            'thumbnail': episode_image,
+            'ext': 'mp3',
+            'vcodec': 'none',
+        }

From 7308b8cb3df5a2df0a86e8050c83b951004a0aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 23:12:13 +0600
Subject: [PATCH 1662/2145] [stitcher] Improve (Closes #7162, closes #7228)

---
 youtube_dl/extractor/stitcher.py | 78 +++++++++++++++++++++++++-------
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index a547debbd..971a1c466 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -1,37 +1,81 @@
-# coding: utf-8
 from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    unescapeHTML,
+)
 
 
 class StitcherIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/[\/a-z\-]+(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/(?:[^/]+/)+e/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+    _TESTS = [{
         'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
         'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
         'info_dict': {
             'id': '40789481',
             'ext': 'mp3',
-            'title': 'Machine Learning Mastery and Cancer Clusters from Talking Machines',
-        }
-    }
+            'title': 'Machine Learning Mastery and Cancer Clusters',
+            'description': 'md5:55163197a44e915a14a1ac3a1de0f2d3',
+            'duration': 1604,
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+    }, {
+        'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
+        'info_dict': {
+            'id': '40846275',
+            'display_id': 'the-rare-hourlong-comedy-plus',
+            'ext': 'mp3',
+            'title': "The CW's 'Crazy Ex-Girlfriend'",
+            'description': 'md5:04f1e2f98eb3f5cbb094cea0f9e19b17',
+            'duration': 2235,
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # escaped title
+        'url': 'http://www.stitcher.com/podcast/marketplace-on-stitcher/e/40910226?autoplay=true',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.stitcher.com/podcast/panoply/getting-in/e/episode-2a-how-many-extracurriculars-should-i-have-40876278?autoplay=true',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        audio_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        audio_id = mobj.group('id')
+        display_id = mobj.group('display_id') or audio_id
 
-        webpage = self._download_webpage(url, audio_id)
+        webpage = self._download_webpage(url, display_id)
 
-        title = self._og_search_title(webpage)
-        url = self._search_regex(r'episodeURL: "(.+?)"', webpage, 'url')
-        episode_image = self._search_regex(r'episodeImage: "(.+?)"', webpage, 'episode_image', fatal=False)
-        duration = int_or_none(self._search_regex(r'duration: (\d+?),', webpage, 'duration', fatal=False))
+        episode = self._parse_json(
+            js_to_json(self._search_regex(
+                r'(?s)var\s+stitcher\s*=\s*({.+?});\n', webpage, 'episode config')),
+            display_id)['config']['episode']
+
+        title = unescapeHTML(episode['title'])
+        formats = [{
+            'url': episode[episode_key],
+            'ext': determine_ext(episode[episode_key]) or 'mp3',
+            'vcodec': 'none',
+        } for episode_key in ('origEpisodeURL', 'episodeURL') if episode.get(episode_key)]
+        description = self._search_regex(
+            r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
+        duration = int_or_none(episode.get('duration'))
+        thumbnail = episode.get('episodeImage')
 
         return {
             'id': audio_id,
-            'url': url,
+            'display_id': display_id,
             'title': title,
+            'description': description,
             'duration': duration,
-            'thumbnail': episode_image,
-            'ext': 'mp3',
-            'vcodec': 'none',
+            'thumbnail': thumbnail,
+            'formats': formats,
         }

From 497f5fd93fe1efd0df8dc58d518c328ed1409457 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 21 Oct 2015 08:24:05 +0100
Subject: [PATCH 1663/2145] [bilibili] extract multiple backup_urls

---
 youtube_dl/extractor/bilibili.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index e00efb6eb..935fcc55c 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -69,21 +69,22 @@ class BiliBiliIE(InfoExtractor):
             pass
 
         doc = ET.fromstring(page)
-        durls = doc.findall('./durl')
 
         entries = []
 
-        for durl in durls:
-            formats = []
-            backup_url = durl.find('./backup_url')
-            if backup_url is not None:
-                formats.append({'url': backup_url.find('./url').text})
+        for durl in doc.findall('./durl'):
             size = durl.find('./filesize|./size')
-            formats.append({
+            formats = [{
                 'url': durl.find('./url').text,
                 'filesize': int_or_none(size.text) if size else None,
                 'ext': 'flv',
-            })
+            }]
+            backup_urls = durl.find('./backup_url')
+            if backup_urls is not None:
+                for backup_url in backup_urls.findall('./url'):
+                    formats.append({'url': backup_url.text})
+            formats.reverse()
+
             entries.append({
                 'id': '%s_part%s' % (cid, durl.find('./order').text),
                 'title': title,

From cc449417c4b3835c31f89e47a5e08e0f0c42ac5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Oct 2015 20:35:22 +0600
Subject: [PATCH 1664/2145] [vine] Use _search_regex for JSON data (Closes
 #7254, closes #7255)

---
 youtube_dl/extractor/vine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index be72f3147..cb2a4b0b5 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -85,8 +85,8 @@ class VineIE(InfoExtractor):
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
         data = self._parse_json(
-            self._html_search_regex(
-                r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+            self._search_regex(
+                r'window\.POST_DATA\s*=\s*{\s*%s\s*:\s*({.+?})\s*};\s*</script>' % video_id,
                 webpage, 'vine data'),
             video_id)
 

From 44d6dd08b299ccf17eb04901cf09a8d333769783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Oct 2015 21:35:57 +0600
Subject: [PATCH 1665/2145] [facebook] Fix extraction (Closes #7252)

---
 youtube_dl/extractor/facebook.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 178a7ca4c..f53c51615 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -14,7 +14,6 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
-    int_or_none,
     limit_length,
     urlencode_postdata,
     get_element_by_id,
@@ -142,16 +141,20 @@ class FacebookIE(InfoExtractor):
         data = dict(json.loads(m.group(1)))
         params_raw = compat_urllib_parse_unquote(data['params'])
         params = json.loads(params_raw)
-        video_data = params['video_data'][0]
 
         formats = []
-        for quality in ['sd', 'hd']:
-            src = video_data.get('%s_src' % quality)
-            if src is not None:
-                formats.append({
-                    'format_id': quality,
-                    'url': src,
-                })
+        for format_id, f in params['video_data'].items():
+            if not f or not isinstance(f, list):
+                continue
+            for quality in ('sd', 'hd'):
+                for src_type in ('src', 'src_no_ratelimit'):
+                    src = f[0].get('%s_%s' % (quality, src_type))
+                    if src:
+                        formats.append({
+                            'format_id': '%s_%s_%s' % (format_id, quality, src_type),
+                            'url': src,
+                            'preference': -10 if format_id == 'progressive' else 0,
+                        })
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
@@ -171,7 +174,5 @@ class FacebookIE(InfoExtractor):
             'id': video_id,
             'title': video_title,
             'formats': formats,
-            'duration': int_or_none(video_data.get('video_duration')),
-            'thumbnail': video_data.get('thumbnail_src'),
             'uploader': uploader,
         }

From 8c3533ba976af15ca9fac8acd68547b195dc4e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Oct 2015 23:57:23 +0200
Subject: [PATCH 1666/2145] [adultswim] Don't default to the native m3u8
 downloader (closes #7243)

Some of the streams are encrypted, which is not supported .
---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 130afe791..3ae618e71 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -183,7 +183,7 @@ class AdultSwimIE(InfoExtractor):
                 media_url = file_el.text
                 if determine_ext(media_url) == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, segment_title, 'mp4', 'm3u8_native', preference=0, m3u8_id='hls'))
+                        media_url, segment_title, 'mp4', preference=0, m3u8_id='hls'))
                 else:
                     formats.append({
                         'format_id': '%s_%s' % (bitrate, ftype),

From 89d5fbf354cf0b49098582a19f76cef67358d375 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 17:47:11 +0800
Subject: [PATCH 1667/2145] [iqiyi] Update key

---
 youtube_dl/extractor/iqiyi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 0e53cb154..2df1da3f0 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,9 +205,9 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-10-10 for Zombie::bite
-        # '7239670519b6ac209a0bee4ef0446a6b24894b8ac2751506e42116212a0d0272e505'[2:66][1::2]
-        enc_key = '97596c0abee04ab49ba25564161ad225'
+        # last update at 2015-10-22 for Zombie::bite
+        # '7223c67061dbea1259d0ceb44f44b6d62288f4f80c972170de5201d2321060270e05'[2:66][0::2]
+        enc_key = '2c76de15dcb44bd28ff0927d50d31620'
         return enc_key
 
     def _real_extract(self, url):

From 7033bc1a5117068c493931cb736d53e68d50f9a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 21:12:29 +0800
Subject: [PATCH 1668/2145] [bbc] Fix test_BBC_9

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 1b3a33e4e..ea67e3f2d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -625,6 +625,7 @@ class BBCIE(BBCCoUkIE):
             'id': 'p02xycnp',
             'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'description': 'BBC Sport\'s David Ornstein has the latest transfer gossip, including rumours of a Manchester United return for Cristiano Ronaldo.',
             'duration': 140,
         },
         'params': {

From a65402ef42c42477f78469f0a6c4af1583d97a31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 21:13:03 +0800
Subject: [PATCH 1669/2145] [bbc.co.uk:article] Add new extractor (#7257)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bbc.py      | 34 ++++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eac5e7d5e..6318ac4a2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -45,6 +45,7 @@ from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbc import (
     BBCCoUkIE,
+    BBCCoUkArticleIE,
     BBCIE,
 )
 from .beeg import BeegIE
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ea67e3f2d..2cdce1eb9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -20,7 +20,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -652,7 +652,7 @@ class BBCIE(BBCCoUkIE):
 
     @classmethod
     def suitable(cls, url):
-        return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+        return False if BBCCoUkIE.suitable(url) or BBCCoUkArticleIE.suitable(url) else super(BBCIE, cls).suitable(url)
 
     def _extract_from_media_meta(self, media_meta, video_id):
         # Direct links to media in media metadata (e.g.
@@ -903,3 +903,33 @@ class BBCIE(BBCCoUkIE):
             })
 
         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
+
+
+class BBCCoUkArticleIE(InfoExtractor):
+    _VALID_URL = 'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    IE_NAME = 'bbc.co.uk:article'
+    IE_DESC = 'BBC articles'
+
+    _TEST = {
+        'url': 'http://www.bbc.co.uk/programmes/articles/3jNQLTMrPlYGTBn0WV6M2MS/not-your-typical-role-model-ada-lovelace-the-19th-century-programmer',
+        'info_dict': {
+            'id': '3jNQLTMrPlYGTBn0WV6M2MS',
+            'title': 'Calculating Ada: The Countess of Computing - Not your typical role model: Ada Lovelace the 19th century programmer - BBC Four',
+            'description': 'Hannah Fry reveals some of her surprising discoveries about Ada Lovelace during filming.',
+        },
+        'playlist_count': 4,
+        'add_ie': ['BBCCoUk'],
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage).strip()
+
+        entries = [self.url_result(programme_url) for programme_url in re.findall(
+            r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 769078755318896fa1a6c5c8aba6f76d6aeddf78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Oct 2015 20:34:11 +0600
Subject: [PATCH 1670/2145] [crunchyroll] Improve subtitle regex (Closes #7262)

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index cecd0c784..f8ce10111 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -245,7 +245,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
-        for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
+        for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
             sub_page = self._download_webpage(
                 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
                 video_id, note='Downloading subtitles for ' + sub_name)

From ab03c0b47c142bfb649eacb3c72ce9cb67184535 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Oct 2015 09:33:05 +0200
Subject: [PATCH 1671/2145] release 2015.10.23

---
 CONTRIBUTING.md        | 3 ++-
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 32c2fd84c..aebded4ce 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -114,12 +114,13 @@ If you want to add support for a new site, you can follow this quick list (assum
             webpage = self._download_webpage(url, video_id)
 
             # TODO more code goes here, for example ...
-            title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+            title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
 
             return {
                 'id': video_id,
                 'title': title,
                 'description': self._og_search_description(webpage),
+                'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cfa665d88..03561b87d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -53,6 +53,7 @@
  - **Bandcamp:album**
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
+ - **bbc.co.uk:article**: BBC articles
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
@@ -515,6 +516,7 @@
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
+ - **Stitcher**
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 660b0050b..d5c7f338d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.18'
+__version__ = '2015.10.23'

From 65d49afa48086b568364bbcbab29feef71031178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:12:46 +0200
Subject: [PATCH 1672/2145] [test/test_download] Use extract_flat =
 'in_playlist' for playlist items

Some playlist extractors return a 'url' result, which wouldn't be resolved.
---
 test/test_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 284418834..a3f1c0644 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -102,7 +102,7 @@ def generator(test_case):
 
         params = get_params(test_case.get('params', {}))
         if is_playlist and 'playlist' not in test_case:
-            params.setdefault('extract_flat', True)
+            params.setdefault('extract_flat', 'in_playlist')
             params.setdefault('skip_download', True)
 
         ydl = YoutubeDL(params, auto_init=False)

From 9170ca5b16f3420892ff06bbe5cccf1679eb75e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:16:08 +0200
Subject: [PATCH 1673/2145] [youtube:channel] Fix test

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 08e821362..bae1b1117 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1644,7 +1644,8 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
         'playlist_mincount': 91,
         'info_dict': {
-            'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+            'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
+            'title': 'Uploads from lex will',
         }
     }]
 

From 5c43afd40f8ba101e0cf90b8fcb5713b378a62c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:23:45 +0200
Subject: [PATCH 1674/2145] [youtube:channel] Support age restricted channels
 (fixes #7277)

---
 youtube_dl/extractor/youtube.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bae1b1117..d7eda7aa7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1647,6 +1647,15 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'Uploads from lex will',
         }
+    }, {
+        'note': 'Age restricted channel',
+        # from https://www.youtube.com/user/DeusExOfficial
+        'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
+        'playlist_mincount': 64,
+        'info_dict': {
+            'id': 'UUs0ifCMCm1icqRbqhUINa0w',
+            'title': 'Uploads from Deus Ex',
+        },
     }]
 
     def _real_extract(self, url):
@@ -1667,7 +1676,7 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
                 'channelId', channel_page, 'channel id', default=None)
             if not channel_playlist_id:
                 channel_playlist_id = self._search_regex(
-                    r'data-channel-external-id="([^"]+)"',
+                    r'data-(?:channel-external-|yt)id="([^"]+)"',
                     channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]

From edeb3e7cb1ab2d82ff7c712a7cc1e338a9dcd8f8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Fri, 23 Oct 2015 15:58:24 +0000
Subject: [PATCH 1675/2145] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a6ec9619c..38db97c59 100644
--- a/README.md
+++ b/README.md
@@ -795,7 +795,7 @@ Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/i
 
 **Please include the full output of youtube-dl when run with `-v`**.
 
-The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 

From dae69640d086ca1e2683ca81b60f48a0c6c83eac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Oct 2015 00:10:28 +0200
Subject: [PATCH 1676/2145] Fix py2exe build (#7276)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4686260e0..bfe931f5b 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ py2exe_options = {
     "compressed": 1,
     "optimize": 2,
     "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe'],
+    "dll_excludes": ['w9xpopen.exe', 'crypt32.dll'],
 }
 
 py2exe_console = [{

From ab9c7214ee6c831a68216caf1cd1f9f3c183e4fd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Oct 2015 00:10:41 +0200
Subject: [PATCH 1677/2145] release 2015.10.24

---
 CONTRIBUTING.md       | 2 +-
 youtube_dl/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index aebded4ce..09ce98ca2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 **Please include the full output of youtube-dl when run with `-v`**.
 
-The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d5c7f338d..125e8ccf5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.23'
+__version__ = '2015.10.24'

From c93153852f342ef26005b37649d58fa944c53fe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:10:53 +0200
Subject: [PATCH 1678/2145] [mitele] Don't encode the URL query (closes #7280)

This seems to produce sporadic errors when trying to access the URL, because on python 3.x when you do '%s' % b'somedata' you get "b'somedata'".
---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 54993e2c9..ccb5c1467 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -56,7 +56,7 @@ class MiTeleIE(InfoExtractor):
                 'sta': '0',
             }
             media = self._download_json(
-                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data)).encode('utf-8')),
+                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data))),
                 display_id, 'Downloading %s JSON' % location['loc'])
             file_ = media.get('file')
             if not file_:

From 6856139705aea86ab1f950c08e605dd47f839be0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:13:26 +0200
Subject: [PATCH 1679/2145] [mitele] Fix test checksum

---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index ccb5c1467..3142fcde2 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -15,7 +15,7 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': 'ace7635b2a0b286aaa37d3ff192d2a8a',
+        'md5': '757b0b66cbd7e0a97226d7d3156cb3e9',
         'info_dict': {
             'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',

From 0198807ef95338bec69cfdcd67249c007e4d4141 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6pmann?=
 <cdoepmann@users.noreply.github.com>
Date: Sat, 24 Oct 2015 11:35:41 +0200
Subject: [PATCH 1680/2145] [spiegeltv] Fix Accept-Encoding issue (server
 chokes on gzip)

---
 youtube_dl/extractor/spiegeltv.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 27f4033c5..a85305281 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -83,6 +83,10 @@ class SpiegeltvIE(InfoExtractor):
                     preference=1,  # Prefer hls since it allows to workaround georestriction
                     m3u8_id='hls', fatal=False)
                 if m3u8_formats is not False:
+                    for m3u8_format in m3u8_formats:
+                        m3u8_format['http_headers'] = {
+                            'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+                        }
                     formats.extend(m3u8_formats)
             else:
                 formats.append({

From 50f01302d347738647262f9442bc4f5d06f013c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:24:08 +0600
Subject: [PATCH 1681/2145] [spiegeltv] Do not extract m3u8 formats since it's
 already a format

---
 youtube_dl/extractor/spiegeltv.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index a85305281..d976bf33c 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,17 +77,16 @@ class SpiegeltvIE(InfoExtractor):
                     'rtmp_live': True,
                 })
             elif determine_ext(endpoint) == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    endpoint.replace('[video]', play_path),
-                    video_id, 'm4v',
-                    preference=1,  # Prefer hls since it allows to workaround georestriction
-                    m3u8_id='hls', fatal=False)
-                if m3u8_formats is not False:
-                    for m3u8_format in m3u8_formats:
-                        m3u8_format['http_headers'] = {
-                            'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
-                        }
-                    formats.extend(m3u8_formats)
+                formats.append({
+                    'url': endpoint.replace('[video]', play_path),
+                    'ext': 'm4v',
+                    'format_id': 'hls',  # Prefer hls since it allows to workaround georestriction
+                    'protocol': 'm3u8',
+                    'preference': 1,
+                    'http_headers': {
+                        'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+                    },
+                })
             else:
                 formats.append({
                     'url': endpoint,

From 943a1e24b896a64c869bbd302f32fe5bd2afec96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:25:04 +0600
Subject: [PATCH 1682/2145] [extractor/common] Use more generic URLError in
 _is_valid_url

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6169fbbeb..720033ddf 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -842,7 +842,7 @@ class InfoExtractor(object):
             self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
+            if isinstance(e.cause, compat_urllib_error.URLError):
                 self.to_screen(
                     '%s: %s URL is invalid, skipping' % (video_id, item))
                 return False

From ac21e7196856d7b689f74ed3f9953cbcbe90bee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:25:44 +0600
Subject: [PATCH 1683/2145] [spiegeltv] Check formats

---
 youtube_dl/extractor/spiegeltv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index d976bf33c..0981e325a 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -91,6 +91,7 @@ class SpiegeltvIE(InfoExtractor):
                 formats.append({
                     'url': endpoint,
                 })
+        self._check_formats(formats, video_id)
 
         thumbnails = []
         for image in media_json['images']:

From 865d1fbafc671815904b2ba3da76544d66c593c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:39:23 +0200
Subject: [PATCH 1684/2145] [extractor/common] Remove unused import

---
 youtube_dl/extractor/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 720033ddf..04b699972 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -16,7 +16,6 @@ from ..compat import (
     compat_cookiejar,
     compat_cookies,
     compat_getpass,
-    compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse,

From 36d72810374ef2dba0232706a461d6dc4aa292d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:41:41 +0200
Subject: [PATCH 1685/2145] [spiegeltv] Fix style issue

Use two spaces before comment.
---
 youtube_dl/extractor/spiegeltv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 0981e325a..034bd47ff 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -84,7 +84,7 @@ class SpiegeltvIE(InfoExtractor):
                     'protocol': 'm3u8',
                     'preference': 1,
                     'http_headers': {
-                        'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+                        'Accept-Encoding': 'deflate',  # gzip causes trouble on the server side
                     },
                 })
             else:

From 7687b354c59efea076fae762206c00de273fbe04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 23 Oct 2015 07:09:41 +0100
Subject: [PATCH 1686/2145] [abc] add support for audio extraction

---
 youtube_dl/extractor/abc.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index f9a389f67..ae80dc529 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -36,6 +36,15 @@ class ABCIE(InfoExtractor):
             'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
         },
         'add_ie': ['Youtube'],
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
+        'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
+        'info_dict': {
+            'id': '6880080',
+            'ext': 'mp3',
+            'title': 'NAB lifts interest rates, following Westpac and CBA',
+            'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
+        },
     }]
 
     def _real_extract(self, url):
@@ -43,7 +52,7 @@ class ABCIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         mobj = re.search(
-            r'inline(?P<type>Video|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
+            r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
             webpage)
         if mobj is None:
             raise ExtractorError('Unable to extract video urls')
@@ -60,11 +69,13 @@ class ABCIE(InfoExtractor):
 
         formats = [{
             'url': url_info['url'],
+            'vcodec': url_info.get('codec') if mobj.group('type') == 'Video' else 'none',
             'width': int_or_none(url_info.get('width')),
             'height': int_or_none(url_info.get('height')),
             'tbr': int_or_none(url_info.get('bitrate')),
             'filesize': int_or_none(url_info.get('filesize')),
         } for url_info in urls_info]
+
         self._sort_formats(formats)
 
         return {

From d97da29da26c920ae31fde94bba5e3b4e1f5a36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:31:42 +0200
Subject: [PATCH 1687/2145] [abc] Support more URL formats

---
 youtube_dl/extractor/abc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index ae80dc529..c0e5d1abf 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
-    _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'http://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
@@ -45,6 +45,9 @@ class ABCIE(InfoExtractor):
             'title': 'NAB lifts interest rates, following Westpac and CBA',
             'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
         },
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 50b936936dcf53b448557c35a90e4678239aaf81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 14:22:47 +0200
Subject: [PATCH 1688/2145] [tutv] Fix test

---
 youtube_dl/extractor/tutv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index fad720b68..822372ea1 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -10,10 +10,10 @@ class TutvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tu\.tv/videos/(?P<id>[^/?]+)'
     _TEST = {
         'url': 'http://tu.tv/videos/robots-futbolistas',
-        'md5': '627c7c124ac2a9b5ab6addb94e0e65f7',
+        'md5': '0cd9e28ad270488911b0d2a72323395d',
         'info_dict': {
             'id': '2973058',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Robots futbolistas',
         },
     }

From 3711304510d3be6a5f9b2b18084aad8687e78001 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 19:35:41 +0100
Subject: [PATCH 1689/2145] [extractor/common] get the redirected m3u8_url in
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 04b699972..10c0d5d1f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -943,13 +943,14 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        m3u8_doc = self._download_webpage(
+        m3u8_doc, urlh = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
         if m3u8_doc is False:
             return m3u8_doc
+        m3u8_url = urlh.geturl()
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 324ac0a243c14340f7e4cd909e2e7c62828a2425 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Sep 2015 20:49:43 +0100
Subject: [PATCH 1690/2145] [downloader/f4m] get the redirected f4m_url and
 handle url query string properly

---
 youtube_dl/downloader/f4m.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 174180db5..b8db6bf9b 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -11,6 +11,7 @@ from .fragment import FragmentFD
 from ..compat import (
     compat_urlparse,
     compat_urllib_error,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     encodeFilename,
@@ -285,7 +286,9 @@ class F4mFD(FragmentFD):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
         self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
-        manifest = self.ydl.urlopen(man_url).read()
+        urlh = self.ydl.urlopen(man_url)
+        man_url = urlh.geturl()
+        manifest = urlh.read()
 
         doc = etree.fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
@@ -329,20 +332,22 @@ class F4mFD(FragmentFD):
         if not live:
             write_metadata_tag(dest_stream, metadata)
 
+        base_url_parsed = compat_urllib_parse_urlparse(base_url)
+
         self._start_frag_download(ctx)
 
         frags_filenames = []
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
-            url = base_url + name
+            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name)
             if akamai_pv:
-                url += '?' + akamai_pv.strip(';')
+                url_parsed = url_parsed._replace(query=url_parsed.query + akamai_pv.strip(';'))
             if info_dict.get('extra_param_to_segment_url'):
-                url += info_dict.get('extra_param_to_segment_url')
+                url_parsed = url_parsed._replace(query=url_parsed.query + info_dict.get('extra_param_to_segment_url'))
             frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = ctx['dl'].download(frag_filename, {'url': url})
+                success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})
                 if not success:
                     return False
                 (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')

From 8cd9614abf81cb41055142d87158b5eda4353a4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:02:31 +0600
Subject: [PATCH 1691/2145] [downloader/f4m] More accurate fragment URL
 construction

---
 youtube_dl/downloader/f4m.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b8db6bf9b..7f6143954 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -340,11 +340,14 @@ class F4mFD(FragmentFD):
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
-            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name)
+            query = []
+            if base_url_parsed.query:
+                query.append(base_url_parsed.query)
             if akamai_pv:
-                url_parsed = url_parsed._replace(query=url_parsed.query + akamai_pv.strip(';'))
+                query.append(akamai_pv.strip(';'))
             if info_dict.get('extra_param_to_segment_url'):
-                url_parsed = url_parsed._replace(query=url_parsed.query + info_dict.get('extra_param_to_segment_url'))
+                query.append(info_dict['extra_param_to_segment_url'])
+            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
             frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
                 success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})

From ec29539e06e156a2bb589af774a80d156b2c2f76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:03:45 +0600
Subject: [PATCH 1692/2145] [senateisvp] Pass extra param as query segment
 without `?`

---
 youtube_dl/extractor/senateisvp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 9c53704ea..474ebb49b 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -121,9 +121,9 @@ class SenateISVPIE(InfoExtractor):
                 'url': compat_urlparse.urljoin(domain, filename) + '?v=3.1.0&fp=&r=&g=',
             }]
         else:
-            hdcore_sign = '?hdcore=3.1.0'
+            hdcore_sign = 'hdcore=3.1.0'
             url_params = (domain, video_id, stream_num)
-            f4m_url = '%s/z/%s_1@%s/manifest.f4m' % url_params + hdcore_sign
+            f4m_url = '%s/z/%s_1@%s/manifest.f4m?' % url_params + hdcore_sign
             m3u8_url = '%s/i/%s_1@%s/master.m3u8' % url_params
             for entry in self._extract_f4m_formats(f4m_url, video_id, f4m_id='f4m'):
                 # URLs without the extra param induce an 404 error

From 8e82ecfe8f0dc2b9dfb6a2cda68e7b5f7926b0e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:04:09 +0600
Subject: [PATCH 1693/2145] [dailymotion] Extract f4m formats

---
 youtube_dl/extractor/dailymotion.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 9cd9ff17d..bc7823931 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -141,9 +141,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     type_ = media.get('type')
                     if type_ == 'application/vnd.lumberjack.manifest':
                         continue
-                    if type_ == 'application/x-mpegURL' or determine_ext(media_url) == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls'))
+                    ext = determine_ext(media_url)
+                    if type_ == 'application/x-mpegURL' or ext == 'm3u8':
+                        m3u8_formats = self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
+                    elif type_ == 'application/f4m' or ext == 'f4m':
+                        f4m_formats = self._extract_f4m_formats(
+                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
                     else:
                         f = {
                             'url': media_url,

From 7e0dc61334cfe9c79e92fd79d9996d191990a80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 20:48:29 +0600
Subject: [PATCH 1694/2145] [njoy] Add support for URLs without display id

---
 youtube_dl/extractor/ndr.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index e3cc6fde8..ba06d8a98 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -14,7 +14,8 @@ from ..utils import (
 
 class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        display_id = next(group for group in mobj.groups() if group)
         webpage = self._download_webpage(url, display_id)
         return self._extract_embed(webpage, display_id)
 
@@ -101,7 +102,7 @@ class NDRIE(NDRBaseIE):
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
@@ -136,6 +137,9 @@ class NJoyIE(NDRBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.n-joy.de/radio/webradio/morningshow209.html',
+        'only_matching': True,
     }]
 
     def _extract_embed(self, webpage, display_id):

From e572a1010b81f5864e610808c86848db4d6ed8e4 Mon Sep 17 00:00:00 2001
From: Erik <erik@vanderleer.nl>
Date: Sat, 17 Oct 2015 19:22:47 +0200
Subject: [PATCH 1695/2145] [youporn] Fix extraction

[youporn] Added description and thumbnail

[youporn] Added uploader and date

[youporn] Removed Try and Except lines

[youporn] Fixed date, fatal, formats and /s*

[youporn] Undid removing comment about video url components & Undid and fixed removal of encrypted URL detection

[youporn] Fix: Add encrypted link to links array only if not already in it

[youporn] Fix: Add encrypted link to links array only if not already in it

[youporn] Fix: cleanup
---
 youtube_dl/extractor/youporn.py | 55 ++++++++++++++-------------------
 1 file changed, 23 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 4ba7c36db..546985f3a 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import json
 import re
 import sys
+import datetime
 
 from .common import InfoExtractor
 from ..compat import (
@@ -27,10 +28,11 @@ class YouPornIE(InfoExtractor):
         'info_dict': {
             'id': '505835',
             'ext': 'mp4',
-            'upload_date': '20101221',
-            'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
-            'uploader': 'Ask Dan And Jennifer',
             'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
+            'description': 'Watch Sex Ed: Is It Safe To Masturbate Daily? at YouPorn.com - YouPorn is the biggest free porn tube site on the net!',
+            'uploader': 'Ask Dan And Jennifer',
+            'thumbnail': 'http://cdn5.image.youporn.phncdn.com/201012/17/505835/640x480/8/sex-ed-is-it-safe-to-masturbate-daily-8.jpg',
+            'date': '20101221',
             'age_limit': 18,
         }
     }
@@ -45,45 +47,34 @@ class YouPornIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
         age_limit = self._rta_search(webpage)
 
-        # Get JSON parameters
-        json_params = self._search_regex(
-            [r'videoJa?son\s*=\s*({.+})',
-             r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
-            webpage, 'JSON parameters')
-        try:
-            params = json.loads(json_params)
-        except ValueError:
-            raise ExtractorError('Invalid JSON')
-
         self.report_extraction(video_id)
-        try:
-            video_title = params['title']
-            upload_date = unified_strdate(params['release_date_f'])
-            video_description = params['description']
-            video_uploader = params['submitted_by']
-            thumbnail = params['thumbnails'][0]['image']
-        except KeyError:
-            raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+        video_title = self._html_search_regex(r'page_params.video_title = \'(.+?)\';', webpage, 'video URL', fatal=False)
+        video_description = self._html_search_meta('description', webpage, 'video DESC', fatal=False)
+        video_thumbnail = self._html_search_regex(r'page_params.imageurl\t=\t"(.+?)";', webpage, 'video THUMB', fatal=False)
+        video_uploader = self._html_search_regex(r"<div class=\'videoInfoBy\'>By:</div>\n<a href=\"[^>]+\">(.+?)</a>", webpage, 'video UPLOADER', fatal=False)
+        video_date = self._html_search_regex(r"<div class='videoInfoTime'>\n<i class='icon-clock'></i> (.+?)\n</div>", webpage, 'video DATE', fatal=False)
+        video_date = datetime.datetime.strptime(video_date, '%B %d, %Y').strftime('%Y%m%d')
 
         # Get all of the links from the page
-        DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
-        download_list_html = self._search_regex(DOWNLOAD_LIST_RE,
-                                                webpage, 'download list').strip()
-        LINK_RE = r'<a href="([^"]+)">'
+        DOWNLOAD_LIST_RE = r'(?s)sources: {\n(?P<download_list>.*?)}'
+        download_list_html = self._search_regex(DOWNLOAD_LIST_RE, webpage, 'download list').strip()
+        LINK_RE = r': \'(.+?)\','
         links = re.findall(LINK_RE, download_list_html)
 
         # Get all encrypted links
-        encrypted_links = re.findall(r'var encryptedQuality[0-9]{3}URL = \'([a-zA-Z0-9+/]+={0,2})\';', webpage)
+        encrypted_links = re.findall(r'page_params.encryptedQuality[0-9]{3,4}URL\s=\s\'([a-zA-Z0-9+/]+={0,2})\';', webpage)
         for encrypted_link in encrypted_links:
             link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
-            links.append(link)
+            # it's unclear if encryted links still differ from normal ones, so only include in links array if it's unique
+            if link not in links:
+                links.append(link)
 
         formats = []
         for link in links:
             # A link looks like this:
-            # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
+            # http://cdn2b.public.youporn.phncdn.com/201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4?rs=200&ri=2500&s=1445599900&e=1445773500&h=5345d19ce9944ec52eb167abf24af248
             # A path looks like this:
-            # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
+            # 201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             video_url = unescapeHTML(link)
             path = compat_urllib_parse_urlparse(video_url).path
             format_parts = path.split('/')[4].split('_')[:2]
@@ -111,11 +102,11 @@ class YouPornIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'uploader': video_uploader,
-            'upload_date': upload_date,
             'title': video_title,
-            'thumbnail': thumbnail,
             'description': video_description,
+            'thumbnail': video_thumbnail,
+            'uploader': video_uploader,
+            'date': video_date,
             'age_limit': age_limit,
             'formats': formats,
         }

From 589c33dadeec18a9d50713a4a200e3e2d9e297bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 22:56:35 +0600
Subject: [PATCH 1696/2145] [youporn] Improve and make more robust (Closes
 #6888, closes #7214)

---
 youtube_dl/extractor/youporn.py | 173 +++++++++++++++++++-------------
 1 file changed, 102 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 546985f3a..d10ebb0bf 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -1,112 +1,143 @@
 from __future__ import unicode_literals
 
-
-import json
 import re
-import sys
-import datetime
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
 from ..utils import (
-    ExtractorError,
+    int_or_none,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
 )
-from ..aes import (
-    aes_decrypt_text
-)
+from ..aes import aes_decrypt_text
 
 
 class YouPornIE(InfoExtractor):
-    _VALID_URL = r'^(?P<proto>https?://)(?:www\.)?(?P<url>youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+))'
+    _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
         'info_dict': {
             'id': '505835',
+            'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
             'ext': 'mp4',
             'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
-            'description': 'Watch Sex Ed: Is It Safe To Masturbate Daily? at YouPorn.com - YouPorn is the biggest free porn tube site on the net!',
+            'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'Ask Dan And Jennifer',
-            'thumbnail': 'http://cdn5.image.youporn.phncdn.com/201012/17/505835/640x480/8/sex-ed-is-it-safe-to-masturbate-daily-8.jpg',
-            'date': '20101221',
+            'upload_date': '20101221',
+            'average_rating': int,
+            'view_count': int,
+            'categories': list,
+            'tags': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-        url = mobj.group('proto') + 'www.' + mobj.group('url')
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
-        age_limit = self._rta_search(webpage)
+        request = compat_urllib_request.Request(url)
+        request.add_header('Cookie', 'age_verified=1')
+        webpage = self._download_webpage(request, display_id)
 
-        self.report_extraction(video_id)
-        video_title = self._html_search_regex(r'page_params.video_title = \'(.+?)\';', webpage, 'video URL', fatal=False)
-        video_description = self._html_search_meta('description', webpage, 'video DESC', fatal=False)
-        video_thumbnail = self._html_search_regex(r'page_params.imageurl\t=\t"(.+?)";', webpage, 'video THUMB', fatal=False)
-        video_uploader = self._html_search_regex(r"<div class=\'videoInfoBy\'>By:</div>\n<a href=\"[^>]+\">(.+?)</a>", webpage, 'video UPLOADER', fatal=False)
-        video_date = self._html_search_regex(r"<div class='videoInfoTime'>\n<i class='icon-clock'></i> (.+?)\n</div>", webpage, 'video DATE', fatal=False)
-        video_date = datetime.datetime.strptime(video_date, '%B %d, %Y').strftime('%Y%m%d')
+        title = self._search_regex(
+            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>.+?)\1',
+             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>([^<])<'],
+            webpage, 'title', group='title')
 
-        # Get all of the links from the page
-        DOWNLOAD_LIST_RE = r'(?s)sources: {\n(?P<download_list>.*?)}'
-        download_list_html = self._search_regex(DOWNLOAD_LIST_RE, webpage, 'download list').strip()
-        LINK_RE = r': \'(.+?)\','
-        links = re.findall(LINK_RE, download_list_html)
+        links = []
 
-        # Get all encrypted links
-        encrypted_links = re.findall(r'page_params.encryptedQuality[0-9]{3,4}URL\s=\s\'([a-zA-Z0-9+/]+={0,2})\';', webpage)
-        for encrypted_link in encrypted_links:
-            link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
-            # it's unclear if encryted links still differ from normal ones, so only include in links array if it's unique
-            if link not in links:
+        sources = self._search_regex(
+            r'sources\s*:\s*({.+?})', webpage, 'sources', default=None)
+        if sources:
+            for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                 links.append(link)
 
+        # Fallback #1
+        for _, link in re.findall(
+                r'(?:videoUrl|videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
+            links.append(link)
+
+        # Fallback #2, this also contains extra low quality 180p format
+        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
+            links.append(link)
+
+        # Fallback #3, encrypted links
+        for _, encrypted_link in re.findall(
+                r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
+            links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))
+
         formats = []
-        for link in links:
-            # A link looks like this:
-            # http://cdn2b.public.youporn.phncdn.com/201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4?rs=200&ri=2500&s=1445599900&e=1445773500&h=5345d19ce9944ec52eb167abf24af248
-            # A path looks like this:
-            # 201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
-            video_url = unescapeHTML(link)
-            path = compat_urllib_parse_urlparse(video_url).path
-            format_parts = path.split('/')[4].split('_')[:2]
-
-            dn = compat_urllib_parse_urlparse(video_url).netloc.partition('.')[0]
-
-            resolution = format_parts[0]
-            height = int(resolution[:-len('p')])
-            bitrate = int(format_parts[1][:-len('k')])
-            format = '-'.join(format_parts) + '-' + dn
-
-            formats.append({
+        for video_url in set(unescapeHTML(link) for link in links):
+            f = {
                 'url': video_url,
-                'format': format,
-                'format_id': format,
-                'height': height,
-                'tbr': bitrate,
-                'resolution': resolution,
-            })
-
+            }
+            # Video URL's path looks like this:
+            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            # We will benefit from it by extracting some metadata
+            mobj = re.search(r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
+            if mobj:
+                height = int(mobj.group('height'))
+                bitrate = int(mobj.group('bitrate'))
+                f.update({
+                    'format_id': '%dp-%dk' % (height, bitrate),
+                    'height': height,
+                    'tbr': bitrate,
+                })
+            formats.append(f)
         self._sort_formats(formats)
 
-        if not formats:
-            raise ExtractorError('ERROR: no known formats available for video')
+        description = self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._search_regex(
+            r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
+            webpage, 'thumbnail', fatal=False, group='thumbnail')
+
+        uploader = self._search_regex(
+            r'<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>\s*<a[^>]+href="[^"]*">([^<]+)</a>',
+            webpage, 'uploader', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',
+            webpage, 'upload date', fatal=False))
+
+        age_limit = self._rta_search(webpage)
+
+        average_rating = int_or_none(self._search_regex(
+            r'<div[^>]+class=["\']videoInfoRating["\'][^>]*>\s*<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
+            webpage, 'average rating', fatal=False))
+
+        view_count = str_to_int(self._search_regex(
+            r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
+            webpage, 'view count', fatal=False))
+
+        def extract_tag_box(title):
+            tag_box = self._search_regex(
+                (r'<div[^>]+class=["\']tagBoxTitle["\'][^>]*>\s*%s\b.*?</div>\s*'
+                 '<div[^>]+class=["\']tagBoxContent["\']>(.+?)</div>') % re.escape(title),
+                webpage, '%s tag box' % title, default=None)
+            if not tag_box:
+                return []
+            return re.findall(r'<a[^>]+href=[^>]+>([^<]+)', tag_box)
+
+        categories = extract_tag_box('Category')
+        tags = extract_tag_box('Tags')
 
         return {
             'id': video_id,
-            'title': video_title,
-            'description': video_description,
-            'thumbnail': video_thumbnail,
-            'uploader': video_uploader,
-            'date': video_date,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'average_rating': average_rating,
+            'view_count': view_count,
+            'categories': categories,
+            'tags': tags,
             'age_limit': age_limit,
             'formats': formats,
         }

From feb7711cf58863a19cae770a878d22a8424e3c61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:01:12 +0600
Subject: [PATCH 1697/2145] [youporn] Make description optional

Some videos does not contain any description
---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index d10ebb0bf..db5d049d2 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -92,7 +92,7 @@ class YouPornIE(InfoExtractor):
 
         description = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None)
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')

From 4f13f8f798be06bc2b3c0c42818bb0785e4cde64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:12:12 +0600
Subject: [PATCH 1698/2145] [youporn] Improve uploader extraction

---
 youtube_dl/extractor/youporn.py | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index db5d049d2..b39fbb5fc 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -15,8 +15,9 @@ from ..aes import aes_decrypt_text
 
 class YouPornIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
+        'md5': '71ec5fcfddacf80f495efa8b6a8d9a89',
         'info_dict': {
             'id': '505835',
             'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
@@ -31,8 +32,29 @@ class YouPornIE(InfoExtractor):
             'categories': list,
             'tags': list,
             'age_limit': 18,
-        }
-    }
+        },
+    }, {
+        # Anonymous User uploader
+        'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
+        'info_dict': {
+            'id': '561726',
+            'display_id': 'big-tits-awesome-brunette-on-amazing-webcam-show',
+            'ext': 'mp4',
+            'title': 'Big Tits Awesome Brunette On amazing webcam show',
+            'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Anonymous User',
+            'upload_date': '20111125',
+            'average_rating': int,
+            'view_count': int,
+            'categories': list,
+            'tags': list,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -97,8 +119,8 @@ class YouPornIE(InfoExtractor):
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
-        uploader = self._search_regex(
-            r'<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>\s*<a[^>]+href="[^"]*">([^<]+)</a>',
+        uploader = self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',

From 7b3a19e5339344037a872574780c39f334cea90e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:17:23 +0600
Subject: [PATCH 1699/2145] [stitcher] Remove origEpisodeURL

It's always 404
---
 youtube_dl/extractor/stitcher.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index 971a1c466..d5c852f52 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -64,7 +64,7 @@ class StitcherIE(InfoExtractor):
             'url': episode[episode_key],
             'ext': determine_ext(episode[episode_key]) or 'mp3',
             'vcodec': 'none',
-        } for episode_key in ('origEpisodeURL', 'episodeURL') if episode.get(episode_key)]
+        } for episode_key in ('episodeURL',) if episode.get(episode_key)]
         description = self._search_regex(
             r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
         duration = int_or_none(episode.get('duration'))

From 755ff8d22ca5607400c1232b194e20a004e4e9eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:41:10 +0600
Subject: [PATCH 1700/2145] [youporn] Extract comment count

---
 youtube_dl/extractor/youporn.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index b39fbb5fc..9bf8d1eeb 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -29,6 +29,7 @@ class YouPornIE(InfoExtractor):
             'upload_date': '20101221',
             'average_rating': int,
             'view_count': int,
+            'comment_count': int,
             'categories': list,
             'tags': list,
             'age_limit': 18,
@@ -47,6 +48,7 @@ class YouPornIE(InfoExtractor):
             'upload_date': '20111125',
             'average_rating': int,
             'view_count': int,
+            'comment_count': int,
             'categories': list,
             'tags': list,
             'age_limit': 18,
@@ -135,6 +137,9 @@ class YouPornIE(InfoExtractor):
         view_count = str_to_int(self._search_regex(
             r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
             webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            r'>All [Cc]omments? \(([\d,.]+)\)',
+            webpage, 'comment count', fatal=False))
 
         def extract_tag_box(title):
             tag_box = self._search_regex(
@@ -158,6 +163,7 @@ class YouPornIE(InfoExtractor):
             'upload_date': upload_date,
             'average_rating': average_rating,
             'view_count': view_count,
+            'comment_count': comment_count,
             'categories': categories,
             'tags': tags,
             'age_limit': age_limit,

From 36e6f62cd0883f0f486d1666d010e5d9e6d515bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Oct 2015 20:04:55 +0100
Subject: [PATCH 1701/2145] Use a wrapper around
 xml.etree.ElementTree.fromstring in python 2.x (#7178)

Attributes aren't unicode objects, so they couldn't be directly used in info_dict fields (for example '--write-description' doesn't work with bytes).
---
 test/test_compat.py                 |  7 +++++++
 test/test_utils.py                  | 11 +++++++----
 youtube_dl/compat.py                | 25 +++++++++++++++++++++++++
 youtube_dl/downloader/f4m.py        |  4 ++--
 youtube_dl/extractor/bbc.py         |  8 +++++---
 youtube_dl/extractor/bilibili.py    |  6 ++++--
 youtube_dl/extractor/brightcove.py  |  4 ++--
 youtube_dl/extractor/common.py      |  4 ++--
 youtube_dl/extractor/crunchyroll.py |  4 ++--
 youtube_dl/extractor/vevo.py        |  6 +++---
 youtube_dl/utils.py                 |  3 ++-
 11 files changed, 61 insertions(+), 21 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 4ee0dc99d..2b0860479 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -13,8 +13,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
+    compat_etree_fromstring,
     compat_expanduser,
     compat_shlex_split,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
 )
@@ -71,5 +73,10 @@ class TestCompat(unittest.TestCase):
     def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
+    def test_compat_etree_fromstring(self):
+        xml = '<el foo="bar"></el>'
+        doc = compat_etree_fromstring(xml.encode('utf-8'))
+        self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index 918a7a9ef..a9e0fed7e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -68,6 +68,9 @@ from youtube_dl.utils import (
     cli_valueless_option,
     cli_bool_option,
 )
+from youtube_dl.compat import (
+    compat_etree_fromstring,
+)
 
 
 class TestUtil(unittest.TestCase):
@@ -242,7 +245,7 @@ class TestUtil(unittest.TestCase):
             <node x="b" y="d" />
             <node x="" />
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
 
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
@@ -263,7 +266,7 @@ class TestUtil(unittest.TestCase):
                 <url>http://server.com/download.mp3</url>
             </media:song>
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
         find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'}))
         self.assertTrue(find('media:song') is not None)
         self.assertEqual(find('media:song/media:author').text, 'The Author')
@@ -285,7 +288,7 @@ class TestUtil(unittest.TestCase):
                 <p>Foo</p>
             </div>
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
         self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
         self.assertEqual(xpath_text(doc, 'div/bar', default='default'), 'default')
         self.assertTrue(xpath_text(doc, 'div/bar') is None)
@@ -297,7 +300,7 @@ class TestUtil(unittest.TestCase):
                 <p x="a">Foo</p>
             </div>
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
         self.assertEqual(xpath_attr(doc, 'div/p', 'x'), 'a')
         self.assertEqual(xpath_attr(doc, 'div/bar', 'x'), None)
         self.assertEqual(xpath_attr(doc, 'div/p', 'y'), None)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index d103ab9ad..cf10835ca 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -14,6 +14,7 @@ import socket
 import subprocess
 import sys
 import itertools
+import xml.etree.ElementTree
 
 
 try:
@@ -212,6 +213,29 @@ try:
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 
+if sys.version_info[0] >= 3:
+    compat_etree_fromstring = xml.etree.ElementTree.fromstring
+else:
+    # on python 2.x the the attributes of a node are str objects instead of
+    # unicode
+    etree = xml.etree.ElementTree
+
+    # on 2.6 XML doesn't have a parser argument, function copied from CPython
+    # 2.7 source
+    def _XML(text, parser=None):
+        if not parser:
+            parser = etree.XMLParser(target=etree.TreeBuilder())
+        parser.feed(text)
+        return parser.close()
+
+    def _element_factory(*args, **kwargs):
+        el = etree.Element(*args, **kwargs)
+        for k, v in el.items():
+            el.set(k, v.decode('utf-8'))
+        return el
+
+    def compat_etree_fromstring(text):
+        return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
@@ -507,6 +531,7 @@ __all__ = [
     'compat_chr',
     'compat_cookiejar',
     'compat_cookies',
+    'compat_etree_fromstring',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7f6143954..6170cc155 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -5,10 +5,10 @@ import io
 import itertools
 import os
 import time
-import xml.etree.ElementTree as etree
 
 from .fragment import FragmentFD
 from ..compat import (
+    compat_etree_fromstring,
     compat_urlparse,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
@@ -290,7 +290,7 @@ class F4mFD(FragmentFD):
         man_url = urlh.geturl()
         manifest = urlh.read()
 
-        doc = etree.fromstring(manifest)
+        doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
                    for f in self._get_unencrypted_media(doc)]
         if requested_bitrate is None:
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2cdce1eb9..a55a6dbc9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,7 +13,10 @@ from ..utils import (
     remove_end,
     unescapeHTML,
 )
-from ..compat import compat_HTTPError
+from ..compat import (
+    compat_etree_fromstring,
+    compat_HTTPError,
+)
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -344,7 +346,7 @@ class BBCCoUkIE(InfoExtractor):
                 url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
+                media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
         return self._process_media_selector(media_selection, programme_id)
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index ecc17ebeb..6c66a1236 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 import itertools
 import json
-import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_etree_fromstring,
+)
 from ..utils import (
     int_or_none,
     unified_strdate,
@@ -88,7 +90,7 @@ class BiliBiliIE(InfoExtractor):
         except ValueError:
             pass
 
-        lq_doc = ET.fromstring(lq_page)
+        lq_doc = compat_etree_fromstring(lq_page)
         lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4721c2293..1686cdde1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 
 import re
 import json
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse,
@@ -119,7 +119,7 @@ class BrightcoveIE(InfoExtractor):
         object_str = fix_xml_ampersands(object_str)
 
         try:
-            object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+            object_doc = compat_etree_fromstring(object_str.encode('utf-8'))
         except compat_xml_parse_error:
             return
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 10c0d5d1f..52523d7b2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -10,7 +10,6 @@ import re
 import socket
 import sys
 import time
-import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
@@ -23,6 +22,7 @@ from ..compat import (
     compat_urllib_request,
     compat_urlparse,
     compat_str,
+    compat_etree_fromstring,
 )
 from ..utils import (
     NO_DEFAULT,
@@ -461,7 +461,7 @@ class InfoExtractor(object):
             return xml_string
         if transform_source:
             xml_string = transform_source(xml_string)
-        return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
+        return compat_etree_fromstring(xml_string.encode('utf-8'))
 
     def _download_json(self, url_or_request, video_id,
                        note='Downloading JSON metadata',
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f8ce10111..0c9b8ca02 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -5,12 +5,12 @@ import re
 import json
 import base64
 import zlib
-import xml.etree.ElementTree
 
 from hashlib import sha1
 from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
@@ -234,7 +234,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         return output
 
     def _extract_subtitles(self, subtitle):
-        sub_root = xml.etree.ElementTree.fromstring(subtitle)
+        sub_root = compat_etree_fromstring(subtitle)
         return [{
             'ext': 'srt',
             'data': self._convert_subtitles_to_srt(sub_root),
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c17094f81..4c0de354f 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
 import re
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_urllib_request,
 )
 from ..utils import (
@@ -97,7 +97,7 @@ class VevoIE(InfoExtractor):
         if last_version['version'] == -1:
             raise ExtractorError('Unable to extract last version of the video')
 
-        renditions = xml.etree.ElementTree.fromstring(last_version['data'])
+        renditions = compat_etree_fromstring(last_version['data'])
         formats = []
         # Already sorted from worst to best quality
         for rend in renditions.findall('rendition'):
@@ -114,7 +114,7 @@ class VevoIE(InfoExtractor):
 
     def _formats_from_smil(self, smil_xml):
         formats = []
-        smil_doc = xml.etree.ElementTree.fromstring(smil_xml.encode('utf-8'))
+        smil_doc = compat_etree_fromstring(smil_xml.encode('utf-8'))
         els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a61e47646..7d846d680 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -36,6 +36,7 @@ import zlib
 from .compat import (
     compat_basestring,
     compat_chr,
+    compat_etree_fromstring,
     compat_html_entities,
     compat_http_client,
     compat_kwargs,
@@ -1974,7 +1975,7 @@ def dfxp2srt(dfxp_data):
 
         return out
 
-    dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
+    dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []
     paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
 

From 387db16a789fea25795433538d80513c18d0f699 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Oct 2015 20:30:54 +0100
Subject: [PATCH 1702/2145] [compat] compat_etree_fromstring: only decode bytes
 objects

---
 test/test_compat.py  | 3 ++-
 youtube_dl/compat.py | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 2b0860479..834f4bc55 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -74,9 +74,10 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
     def test_compat_etree_fromstring(self):
-        xml = '<el foo="bar"></el>'
+        xml = '<el foo="bar" spam="中文"></el>'
         doc = compat_etree_fromstring(xml.encode('utf-8'))
         self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
+        self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index cf10835ca..f39d4e9a9 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,8 +216,7 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the the attributes of a node are str objects instead of
-    # unicode
+    # on python 2.x the the attributes of a node aren't always unicode objects
     etree = xml.etree.ElementTree
 
     # on 2.6 XML doesn't have a parser argument, function copied from CPython
@@ -231,7 +230,8 @@ else:
     def _element_factory(*args, **kwargs):
         el = etree.Element(*args, **kwargs)
         for k, v in el.items():
-            el.set(k, v.decode('utf-8'))
+            if isinstance(v, bytes):
+                el.set(k, v.decode('utf-8'))
         return el
 
     def compat_etree_fromstring(text):

From 5f9f87c06fa819c75e59f5d0d491d191c229abbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Mon, 26 Oct 2015 14:42:17 +0100
Subject: [PATCH 1703/2145] [vidme] Check for deleted videos

---
 youtube_dl/extractor/vidme.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index eb5cde761..3d63ed4f0 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -101,6 +101,10 @@ class VidmeIE(InfoExtractor):
         # suspended
         'url': 'https://vid.me/Ox3G',
         'only_matching': True,
+    }, {
+        # deleted
+        'url': 'https://vid.me/KTPm',
+        'only_matching': True,
     }, {
         # no formats in the API response
         'url': 'https://vid.me/e5g',
@@ -143,6 +147,11 @@ class VidmeIE(InfoExtractor):
 
         video = response['video']
 
+        if video.get('state') == 'deleted':
+            raise ExtractorError(
+                'Vidme said: Sorry, this video has been deleted.',
+                expected=True)
+
         if video.get('state') in ('user-disabled', 'suspended'):
             raise ExtractorError(
                 'Vidme said: This video has been suspended either due to a copyright claim, '

From 5dadae079bd053c822353b081e94d9daff333208 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Mon, 26 Oct 2015 15:11:09 +0100
Subject: [PATCH 1704/2145] [francetv] Add subtitles support

---
 youtube_dl/extractor/francetv.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 129984a5f..eaaa43958 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -83,6 +83,16 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         if subtitle:
             title += ' - %s' % subtitle
 
+        subtitles = {}
+        for subtitle_accessibilite in info['subtitles']:
+            if subtitle_accessibilite['url'] is not '':
+                if not subtitles:
+                    subtitles['fr'] = []
+                subtitles['fr'].append({
+                    'ext': subtitle_accessibilite['format'],
+                    'url': subtitle_accessibilite['url'],
+                })
+
         return {
             'id': video_id,
             'title': title,
@@ -91,6 +101,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 6e4b8b28916aaafc6d1b4b4d69a6f667e35d413f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Oct 2015 20:35:28 +0600
Subject: [PATCH 1705/2145] [francetv] Make subtitles more robust (Closes
 #7298)

---
 youtube_dl/extractor/francetv.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index eaaa43958..07115b9d4 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -84,14 +84,12 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             title += ' - %s' % subtitle
 
         subtitles = {}
-        for subtitle_accessibilite in info['subtitles']:
-            if subtitle_accessibilite['url'] is not '':
-                if not subtitles:
-                    subtitles['fr'] = []
-                subtitles['fr'].append({
-                    'ext': subtitle_accessibilite['format'],
-                    'url': subtitle_accessibilite['url'],
-                })
+        subtitles_list = [{
+            'url': subtitle['url'],
+            'ext': subtitle.get('format'),
+        } for subtitle in info.get('subtitles', []) if subtitle.get('url')]
+        if subtitles_list:
+            subtitles['fr'] = subtitles_list
 
         return {
             'id': video_id,

From c137cc0d33fee8369b857c8f12a9116379248127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Oct 2015 20:35:45 +0600
Subject: [PATCH 1706/2145] [francetv] Add subtitles test

---
 youtube_dl/extractor/francetv.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 07115b9d4..a31cc3c97 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -129,6 +129,9 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'title': 'Soir 3',
             'upload_date': '20130826',
             'timestamp': 1377548400,
+            'subtitles': {
+                'fr': 'mincount:2',
+            },
         },
     }, {
         'url': 'http://www.francetvinfo.fr/elections/europeennes/direct-europeennes-regardez-le-debat-entre-les-candidats-a-la-presidence-de-la-commission_600639.html',

From f78546272cf7c4b10c8003870728ab69bec982fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 26 Oct 2015 16:41:24 +0100
Subject: [PATCH 1707/2145] [compat] compat_etree_fromstring: also decode the
 text attribute

Deletes parse_xml from utils, because it also does it.
---
 test/test_compat.py             | 11 ++++++++++-
 youtube_dl/compat.py            | 18 ++++++++++++++++--
 youtube_dl/extractor/ard.py     |  4 ++--
 youtube_dl/extractor/generic.py |  4 ++--
 youtube_dl/utils.py             | 23 -----------------------
 5 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 834f4bc55..b6bfad05e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -74,10 +74,19 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
     def test_compat_etree_fromstring(self):
-        xml = '<el foo="bar" spam="中文"></el>'
+        xml = '''
+            <root foo="bar" spam="中文">
+                <normal>foo</normal>
+                <chinese>中文</chinese>
+                <foo><bar>spam</bar></foo>
+            </root>
+        '''
         doc = compat_etree_fromstring(xml.encode('utf-8'))
         self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
         self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
+        self.assertTrue(isinstance(doc.find('normal').text, compat_str))
+        self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
+        self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f39d4e9a9..2d43ec852 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,9 +216,19 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the the attributes of a node aren't always unicode objects
+    # on python 2.x the attributes and text of a node aren't always unicode
+    # objects
     etree = xml.etree.ElementTree
 
+    try:
+        _etree_iter = etree.Element.iter
+    except AttributeError:  # Python <=2.6
+        def _etree_iter(root):
+            for el in root.findall('*'):
+                yield el
+                for sub in _etree_iter(el):
+                    yield sub
+
     # on 2.6 XML doesn't have a parser argument, function copied from CPython
     # 2.7 source
     def _XML(text, parser=None):
@@ -235,7 +245,11 @@ else:
         return el
 
     def compat_etree_fromstring(text):
-        return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        for el in _etree_iter(doc):
+            if el.text is not None and isinstance(el.text, bytes):
+                el.text = el.text.decode('utf-8')
+        return doc
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6f465789b..73be6d204 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -14,8 +14,8 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     xpath_text,
-    parse_xml,
 )
+from ..compat import compat_etree_fromstring
 
 
 class ARDMediathekIE(InfoExtractor):
@@ -161,7 +161,7 @@ class ARDMediathekIE(InfoExtractor):
             raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True)
 
         if re.search(r'[\?&]rss($|[=&])', url):
-            doc = parse_xml(webpage)
+            doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
                 return GenericIE()._extract_rss(url, video_id, doc)
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..1de96b268 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -9,6 +9,7 @@ import sys
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
+    compat_etree_fromstring,
     compat_urllib_parse_unquote,
     compat_urllib_request,
     compat_urlparse,
@@ -21,7 +22,6 @@ from ..utils import (
     HEADRequest,
     is_html,
     orderedSet,
-    parse_xml,
     smuggle_url,
     unescapeHTML,
     unified_strdate,
@@ -1237,7 +1237,7 @@ class GenericIE(InfoExtractor):
 
         # Is it an RSS feed, a SMIL file or a XSPF playlist?
         try:
-            doc = parse_xml(webpage)
+            doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
                 return self._extract_rss(url, video_id, doc)
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7d846d680..c761ea22a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1652,29 +1652,6 @@ def encode_dict(d, encoding='utf-8'):
     return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
 
 
-try:
-    etree_iter = xml.etree.ElementTree.Element.iter
-except AttributeError:  # Python <=2.6
-    etree_iter = lambda n: n.findall('.//*')
-
-
-def parse_xml(s):
-    class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
-        def doctype(self, name, pubid, system):
-            pass  # Ignore doctypes
-
-    parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
-    kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
-    tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
-    # Fix up XML parser in Python 2.x
-    if sys.version_info < (3, 0):
-        for n in etree_iter(tree):
-            if n.text is not None:
-                if not isinstance(n.text, compat_str):
-                    n.text = n.text.decode('utf-8')
-    return tree
-
-
 US_RATINGS = {
     'G': 0,
     'PG': 10,

From a526167d40983e47231d10c09c9f9064e0298604 Mon Sep 17 00:00:00 2001
From: Pierre Fenoll <pierrefenoll@gmail.com>
Date: Tue, 27 Oct 2015 11:58:59 +0100
Subject: [PATCH 1708/2145] [francetv] Accept mobile URLs

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index a31cc3c97..d63dc4d7c 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -105,7 +105,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 class PluzzIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'pluzz.francetv.fr'
-    _VALID_URL = r'https?://pluzz\.francetv\.fr/videos/(.*?)\.html'
+    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(.*?)\.html'
 
     # Can't use tests, videos expire in 7 days
 

From 0a192fbea798c843ad6fef37106901d431f39b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Oct 2015 21:43:29 +0600
Subject: [PATCH 1709/2145] [pluzz] Fix mobile support and modernize (Closes
 #7305)

---
 youtube_dl/extractor/francetv.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d63dc4d7c..00a80ba61 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -105,15 +105,21 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 class PluzzIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'pluzz.francetv.fr'
-    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(.*?)\.html'
+    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(?P<id>.+?)\.html'
 
     # Can't use tests, videos expire in 7 days
 
     def _real_extract(self, url):
-        title = re.match(self._VALID_URL, url).group(1)
-        webpage = self._download_webpage(url, title)
-        video_id = self._search_regex(
-            r'data-diffusion="(\d+)"', webpage, 'ID')
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_meta(
+            'id_video', webpage, 'video id', default=None)
+        if not video_id:
+            video_id = self._search_regex(
+            r'data-diffusion=["\'](\d+)', webpage, 'video id')
+
         return self._extract_video(video_id, 'Pluzz')
 
 

From 7ccb2b84ddb65f41d01037b5b62301886be9d22c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 28 Oct 2015 08:22:04 +0100
Subject: [PATCH 1710/2145] [francetv] fix style issues reported by flake8

* Don't redefine variable in list comprehension
* Line missing indentation
---
 youtube_dl/extractor/francetv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 00a80ba61..8e60cf60f 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -85,9 +85,9 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
         subtitles = {}
         subtitles_list = [{
-            'url': subtitle['url'],
-            'ext': subtitle.get('format'),
-        } for subtitle in info.get('subtitles', []) if subtitle.get('url')]
+            'url': subformat['url'],
+            'ext': subformat.get('format'),
+        } for subformat in info.get('subtitles', []) if subformat.get('url')]
         if subtitles_list:
             subtitles['fr'] = subtitles_list
 
@@ -118,7 +118,7 @@ class PluzzIE(FranceTVBaseInfoExtractor):
             'id_video', webpage, 'video id', default=None)
         if not video_id:
             video_id = self._search_regex(
-            r'data-diffusion=["\'](\d+)', webpage, 'video id')
+                r'data-diffusion=["\'](\d+)', webpage, 'video id')
 
         return self._extract_video(video_id, 'Pluzz')
 

From 4e16c1f80b009001aaea6f8baca5dfbfa9b58c11 Mon Sep 17 00:00:00 2001
From: Cian Ruane <CianLR@users.noreply.github.com>
Date: Fri, 16 Oct 2015 01:23:09 +0100
Subject: [PATCH 1711/2145] [clyp] Add extractor

Update __init__.py

[clyp.it] Extract ID idiomatically and make duration and description optional
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/clyp.py     | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/clyp.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6318ac4a2..f98e6487e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,6 +90,7 @@ from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
 from .cloudy import CloudyIE
 from .clubic import ClubicIE
+from .clyp import ClypIE
 from .cmt import CMTIE
 from .cnet import CNETIE
 from .cnn import (
diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
new file mode 100644
index 000000000..906729b30
--- /dev/null
+++ b/youtube_dl/extractor/clyp.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ClypIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
+
+    _TESTS = [{
+        'url': 'https://clyp.it/ojz2wfah',
+        'md5': '1d4961036c41247ecfdcc439c0cddcbb',
+        'info_dict': {
+            'id': 'ojz2wfah',
+            'ext': 'mp3',
+            'title': 'Krisson80 - bits wip wip',
+            'description': '#Krisson80BitsWipWip #chiptune\n#wip',
+        },
+    }, {
+        'url': 'https://clyp.it/ojz2wfah',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        api_url = 'https://api.clyp.it/' + audio_id
+        metadata = self._download_json(api_url, audio_id)
+
+        title = metadata['Title']
+
+        description = None
+        if metadata['Description']: description = metadata['Description']
+
+        duration = None
+        if metadata['Duration']: duration = int(metadata['Duration'])
+        
+        formats = [
+            {
+            'url': metadata['OggUrl'],
+            'format_id': 'ogg',
+            'preference': -2
+            },{
+            'url': metadata['Mp3Url'],
+            'format_id': 'mp3',
+            'preference': -1
+            }]
+
+        return {
+            'id': audio_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'duration': duration
+        }

From 52c3a6e49d2cbc1932992d816d28bbed629daadc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Oct 2015 21:40:22 +0600
Subject: [PATCH 1712/2145] [utils] Improve parse_iso8601

---
 test/test_utils.py  |  2 ++
 youtube_dl/utils.py | 13 +++++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 918a7a9ef..0c34f0e55 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -425,6 +425,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)
+        self.assertEqual(parse_iso8601('2015-09-29T08:27:31.727'), 1443515251)
+        self.assertEqual(parse_iso8601('2015-09-29T08-27-31.727'), None)
 
     def test_strip_jsonp(self):
         stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a61e47646..558c9c7d5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -814,9 +814,11 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
     if date_str is None:
         return None
 
+    date_str = re.sub(r'\.[0-9]+', '', date_str)
+
     if timezone is None:
         m = re.search(
-            r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+            r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
             date_str)
         if not m:
             timezone = datetime.timedelta()
@@ -829,9 +831,12 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
                 timezone = datetime.timedelta(
                     hours=sign * int(m.group('hours')),
                     minutes=sign * int(m.group('minutes')))
-    date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
-    dt = datetime.datetime.strptime(date_str, date_format) - timezone
-    return calendar.timegm(dt.timetuple())
+    try:
+        date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
+        dt = datetime.datetime.strptime(date_str, date_format) - timezone
+        return calendar.timegm(dt.timetuple())
+    except ValueError:
+        pass
 
 
 def unified_strdate(date_str, day_first=True):

From 03c2c162f9eaac6b474a1be9e985621f5b7b8c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Oct 2015 21:42:01 +0600
Subject: [PATCH 1713/2145] [clyp] Improve and cleanup (Closes #7194)

---
 youtube_dl/extractor/clyp.py | 62 ++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
index 906729b30..57e643799 100644
--- a/youtube_dl/extractor/clyp.py
+++ b/youtube_dl/extractor/clyp.py
@@ -1,16 +1,15 @@
-# coding: utf-8
-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    parse_iso8601,
+)
 
 
 class ClypIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
-
-    _TESTS = [{
+    _TEST = {
         'url': 'https://clyp.it/ojz2wfah',
         'md5': '1d4961036c41247ecfdcc439c0cddcbb',
         'info_dict': {
@@ -18,40 +17,41 @@ class ClypIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'Krisson80 - bits wip wip',
             'description': '#Krisson80BitsWipWip #chiptune\n#wip',
+            'duration': 263.21,
+            'timestamp': 1443515251,
+            'upload_date': '20150929',
         },
-    }, {
-        'url': 'https://clyp.it/ojz2wfah',
-        'only_matching': True,
-    }]
+    }
 
     def _real_extract(self, url):
         audio_id = self._match_id(url)
-        api_url = 'https://api.clyp.it/' + audio_id
-        metadata = self._download_json(api_url, audio_id)
+
+        metadata = self._download_json(
+            'https://api.clyp.it/%s' % audio_id, audio_id)
+
+        formats = []
+        for secure in ('', 'Secure'):
+            for ext in ('Ogg', 'Mp3'):
+                format_id = '%s%s' % (secure, ext)
+                format_url = metadata.get('%sUrl' % format_id)
+                if format_url:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                        'vcodec': 'none',
+                    })
+        self._sort_formats(formats)
 
         title = metadata['Title']
-
-        description = None
-        if metadata['Description']: description = metadata['Description']
-
-        duration = None
-        if metadata['Duration']: duration = int(metadata['Duration'])
-        
-        formats = [
-            {
-            'url': metadata['OggUrl'],
-            'format_id': 'ogg',
-            'preference': -2
-            },{
-            'url': metadata['Mp3Url'],
-            'format_id': 'mp3',
-            'preference': -1
-            }]
+        description = metadata.get('Description')
+        duration = float_or_none(metadata.get('Duration'))
+        timestamp = parse_iso8601(metadata.get('DateCreated'))
 
         return {
             'id': audio_id,
             'title': title,
-            'formats': formats,
             'description': description,
-            'duration': duration
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
         }

From ae37338e681319a28d98dc551253d9fa1830969a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Oct 2015 13:58:40 +0100
Subject: [PATCH 1714/2145] [compat] compat_etree_fromstring: clarify comment

---
 youtube_dl/compat.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2d43ec852..a3e85264a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,8 +216,8 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the attributes and text of a node aren't always unicode
-    # objects
+    # python 2.x tries to encode unicode strings with ascii (see the
+    # XMLParser._fixtext method)
     etree = xml.etree.ElementTree
 
     try:

From 6fb8ace671db2f2bdcc9cd7ac6b9f81fbd356791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Oct 2015 22:44:01 +0600
Subject: [PATCH 1715/2145] [moniker] Add support for builtin embedded videos
 (Closes #7244)

---
 youtube_dl/extractor/moniker.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 69e4bcd1a..204c03c4a 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class MonikerIE(InfoExtractor):
     IE_DESC = 'allmyvideos.net and vidspot.net'
-    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?:(?:2|v)/v-)?(?P<id>[a-zA-Z0-9_-]+)'
 
     _TESTS = [{
         'url': 'http://allmyvideos.net/jih3nce3x6wn',
@@ -64,18 +64,30 @@ class MonikerIE(InfoExtractor):
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
-        data = dict(fields)
+        builtin_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>.+?/builtin-.+?)\1',
+            orig_webpage, 'builtin URL', default=None, group='url')
 
-        post = compat_urllib_parse.urlencode(data)
-        headers = {
-            b'Content-Type': b'application/x-www-form-urlencoded',
-        }
-        req = compat_urllib_request.Request(url, post, headers)
-        webpage = self._download_webpage(
-            req, video_id, note='Downloading video page ...')
+        if builtin_url:
+            req = compat_urllib_request.Request(builtin_url)
+            req.add_header('Referer', url)
+            webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
+            title = self._og_search_title(orig_webpage).strip()
+            description = self._og_search_description(orig_webpage).strip()
+        else:
+            fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
+            data = dict(fields)
 
-        title = os.path.splitext(data['fname'])[0]
+            post = compat_urllib_parse.urlencode(data)
+            headers = {
+                b'Content-Type': b'application/x-www-form-urlencoded',
+            }
+            req = compat_urllib_request.Request(url, post, headers)
+            webpage = self._download_webpage(
+                req, video_id, note='Downloading video page ...')
+
+            title = os.path.splitext(data['fname'])[0]
+            description = None
 
         # Could be several links with different quality
         links = re.findall(r'"file" : "?(.+?)",', webpage)
@@ -89,5 +101,6 @@ class MonikerIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
+            'description': description,
             'formats': formats,
         }

From 721f5a277ca0012ee72c9d4b3e5550e52a0a596d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Oct 2015 22:47:18 +0600
Subject: [PATCH 1716/2145] [moniker] Add tests for #7244

---
 youtube_dl/extractor/moniker.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 204c03c4a..7c0c4e50e 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -46,6 +46,18 @@ class MonikerIE(InfoExtractor):
     }, {
         'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
         'only_matching': True,
+    }, {
+        'url': 'http://vidspot.net/2/v-ywDf99',
+        'md5': '5f8254ce12df30479428b0152fb8e7ba',
+        'info_dict': {
+            'id': 'ywDf99',
+            'ext': 'mp4',
+            'title': 'IL FAIT LE MALIN EN PORSHE CAYENNE ( mais pas pour longtemps)',
+            'description': 'IL FAIT LE MALIN EN PORSHE CAYENNE.',
+        },
+    }, {
+        'url': 'http://allmyvideos.net/v/v-HXZm5t',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 30bd1c16c8e2a767246cc115f12c1d2e99e2f8ec Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 29 Oct 2015 19:44:26 +0100
Subject: [PATCH 1717/2145] [adobetv] use api for extraction and add support
 specific language videos

---
 youtube_dl/extractor/adobetv.py | 60 +++++++++++++--------------------
 1 file changed, 24 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 5e43adc51..3d25caad6 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -1,23 +1,26 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     unified_strdate,
     str_to_int,
+    int_or_none,
     float_or_none,
     ISO639Utils,
 )
 
 
 class AdobeTVIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.adobe\.com/watch/[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
         'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
         'info_dict': {
-            'id': 'quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop',
+            'id': '10981',
             'ext': 'mp4',
             'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
             'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
@@ -29,46 +32,31 @@ class AdobeTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        language, show_urlname, urlname = re.match(self._VALID_URL, url).groups()
+        if not language:
+            language = 'en'
 
-        player = self._parse_json(
-            self._search_regex(r'html5player:\s*({.+?})\s*\n', webpage, 'player'),
-            video_id)
-
-        title = player.get('title') or self._search_regex(
-            r'data-title="([^"]+)"', webpage, 'title')
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
-        upload_date = unified_strdate(
-            self._html_search_meta('datepublished', webpage, 'upload date'))
-
-        duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration') or
-            self._search_regex(
-                r'Runtime:\s*(\d{2}:\d{2}:\d{2})',
-                webpage, 'duration', fatal=False))
-
-        view_count = str_to_int(self._search_regex(
-            r'<div class="views">\s*Views?:\s*([\d,.]+)\s*</div>',
-            webpage, 'view count'))
+        video_data = self._download_json(
+            'http://tv.adobe.com/api/v4/episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
+            urlname)['data'][0]
 
         formats = [{
-            'url': source['src'],
-            'format_id': source.get('quality') or source['src'].split('-')[-1].split('.')[0] or None,
-            'tbr': source.get('bitrate'),
-        } for source in player['sources']]
+            'url': source['url'],
+            'format_id': source.get('quality_level') or source['url'].split('-')[-1].split('.')[0] or None,
+            'width': int_or_none(source.get('width')),
+            'height': int_or_none(source.get('height')),
+            'tbr': int_or_none(source.get('video_data_rate')),
+        } for source in video_data['videos']]
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'duration': duration,
-            'view_count': view_count,
+            'id': str(video_data['id']),
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnail'),
+            'upload_date': unified_strdate(video_data.get('start_date')),
+            'duration': parse_duration(video_data.get('duration')),
+            'view_count': str_to_int(video_data.get('playcount')),
             'formats': formats,
         }
 

From 402ca40c9d0b20f1d04a0035b4cda0cc1184c689 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 29 Oct 2015 19:55:04 +0100
Subject: [PATCH 1718/2145] [adobetv] extract AdobeTVVideo info from json
 directly

---
 youtube_dl/extractor/adobetv.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 3d25caad6..383c89485 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -10,6 +10,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     ISO639Utils,
+    determine_ext,
 )
 
 
@@ -79,28 +80,25 @@ class AdobeTVVideoIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        player_params = self._parse_json(self._search_regex(
-            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'player parameters'),
-            video_id)
+        video_data = self._download_json(url + '?format=json', video_id)
 
         formats = [{
+            'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')),
             'url': source['src'],
-            'width': source.get('width'),
-            'height': source.get('height'),
-            'tbr': source.get('bitrate'),
-        } for source in player_params['sources']]
+            'width': int_or_none(source.get('width')),
+            'height': int_or_none(source.get('height')),
+            'tbr': int_or_none(source.get('bitrate')),
+        } for source in video_data['sources']]
+        self._sort_formats(formats)
 
         # For both metadata and downloaded files the duration varies among
         # formats. I just pick the max one
         duration = max(filter(None, [
             float_or_none(source.get('duration'), scale=1000)
-            for source in player_params['sources']]))
+            for source in video_data['sources']]))
 
         subtitles = {}
-        for translation in player_params.get('translations', []):
+        for translation in video_data.get('translations', []):
             lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
             if lang_id not in subtitles:
                 subtitles[lang_id] = []
@@ -112,8 +110,9 @@ class AdobeTVVideoIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': player_params['title'],
-            'description': self._og_search_description(webpage),
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': video_data['video'].get('poster'),
             'duration': duration,
             'subtitles': subtitles,
         }

From 9a605c8859e5ecf164719b890ea62b76afb0b874 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 29 Oct 2015 20:00:27 +0100
Subject: [PATCH 1719/2145] [adobetv] add support for show and channel
 extraction

---
 youtube_dl/extractor/__init__.py |  2 +
 youtube_dl/extractor/adobetv.py  | 72 ++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6318ac4a2..d4b42dc25 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -6,6 +6,8 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import (
     AdobeTVIE,
+    AdobeTVShowIE,
+    AdobeTVChannelIE,
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 383c89485..d0bfafa45 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -62,6 +62,78 @@ class AdobeTVIE(InfoExtractor):
         }
 
 
+class AdobeTVPlaylistBaseIE(InfoExtractor):
+    def _parse_page_data(self, page_data):
+        return [self.url_result(self._get_element_url(element_data)) for element_data in page_data]
+
+    def _extract_playlist_entries(self, url, display_id):
+        page = self._download_json(url, display_id)
+        entries = self._parse_page_data(page['data'])
+        for page_num in range(2, page['paging']['pages'] + 1):
+            entries.extend(self._parse_page_data(
+                self._download_json(url + '&page=%d' % page_num, display_id)['data']))
+        return entries
+
+
+class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
+    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?show/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
+        'info_dict': {
+            'id': '36',
+            'title': 'The Complete Picture with Julieanne Kost',
+            'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
+        },
+        'playlist_mincount': 136,
+    }
+
+    def _get_element_url(self, element_data):
+        return element_data['urls'][0]
+
+    def _real_extract(self, url):
+        language, show_urlname = re.match(self._VALID_URL, url).groups()
+        if not language:
+            language = 'en'
+        query = 'language=%s&show_urlname=%s' % (language, show_urlname)
+
+        show_data = self._download_json(
+            'http://tv.adobe.com/api/v4/show/get/?%s' % query, show_urlname)['data'][0]
+
+        return self.playlist_result(
+            self._extract_playlist_entries('http://tv.adobe.com/api/v4/episode/?%s' % query, show_urlname),
+            str(show_data['id']),
+            show_data['show_name'],
+            show_data['show_description'])
+
+
+class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
+    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'
+
+    _TEST = {
+        'url': 'http://tv.adobe.com/channel/development',
+        'info_dict': {
+            'id': 'development',
+        },
+        'playlist_mincount': 96,
+    }
+
+    def _get_element_url(self, element_data):
+        return element_data['url']
+
+    def _real_extract(self, url):
+        language, channel_urlname, category_urlname = re.match(self._VALID_URL, url).groups()
+        if not language:
+            language = 'en'
+        query = 'language=%s&channel_urlname=%s' % (language, channel_urlname)
+        if category_urlname:
+            query += '&category_urlname=%s' % category_urlname
+
+        return self.playlist_result(
+            self._extract_playlist_entries('http://tv.adobe.com/api/v4/show/?%s' % query, channel_urlname),
+            channel_urlname)
+
+
 class AdobeTVVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
 

From 6722ebd43720e836af8217fa078fa1a604b98229 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Oct 2015 21:00:36 +0600
Subject: [PATCH 1720/2145] [anitube] Relax key regex (Closes #7303)

 Another variant seen http://anitubebr.xpg.uol.com.br/embed/
---
 youtube_dl/extractor/anitube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 31f0d417c..23f942ae2 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -26,8 +26,8 @@ class AnitubeIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
-        key = self._html_search_regex(
-            r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', webpage, 'key')
+        key = self._search_regex(
+            r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
 
         config_xml = self._download_xml(
             'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key)

From 240384afe6983b71d3a6cb8891312185a15619f6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 20:06:38 +0100
Subject: [PATCH 1721/2145] [clipfish] improve info extraction

---
 youtube_dl/extractor/clipfish.py | 56 ++++++++++++--------------------
 1 file changed, 21 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 7af903571..d142e326f 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -4,11 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     int_or_none,
-    js_to_json,
-    parse_iso8601,
-    remove_end,
+    unified_strdate,
 )
 
 
@@ -21,48 +18,37 @@ class ClipfishIE(InfoExtractor):
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
-            'timestamp': 1370938118,
+            'description': 'Video zu FIFA 14: E3 2013 Trailer',
             'upload_date': '20130611',
             'duration': 82,
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        video_info = self._download_json('http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id, video_id)['items'][0]
 
-        video_info = self._parse_json(
-            js_to_json(self._html_search_regex(
-                '(?s)videoObject\s*=\s*({.+?});', webpage, 'video object')),
-            video_id)
-
-        formats = []
-        for video_url in re.findall(r'var\s+videourl\s*=\s*"([^"]+)"', webpage):
-            ext = determine_ext(video_url)
-            if ext == 'm3u8':
-                formats.append({
-                    'url': video_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
-                    'ext': 'mp4',
-                    'format_id': 'hls',
-                })
-            else:
-                formats.append({
-                    'url': video_url,
-                    'format_id': ext,
-                })
-        self._sort_formats(formats)
-
-        title = remove_end(self._og_search_title(webpage), ' - Video')
-        thumbnail = self._og_search_thumbnail(webpage)
-        duration = int_or_none(video_info.get('length'))
-        timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage, 'upload date'))
+        formats = [{
+            'url': video_info['media_videourl_hls'].replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+            'ext': 'mp4',
+            'format_id': 'hls',
+        },{
+            'url': video_info['media_videourl'],
+            'format_id': 'mp4',
+            'width': int_or_none(video_info.get('width')),
+            'height': int_or_none(video_info.get('height')),
+            'tbr': int_or_none(video_info.get('bitrate')),
+        }]
 
         return {
             'id': video_id,
-            'title': title,
+            'title': video_info['title'],
+            'description': video_info.get('descr'),
             'formats': formats,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
+            'thumbnail': video_info.get('media_content_thumbnail_large') or video_info.get('media_thumbnail'),
+            'duration': int_or_none(video_info.get('media_length')),
+            'upload_date': unified_strdate(video_info.get('pubDate')),
+            'view_count': int_or_none(video_info.get('media_views'))
         }

From 47f2d01a5ac074f6959aa12e8bc00310f18a54e8 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Thu, 24 Sep 2015 22:19:09 +0200
Subject: [PATCH 1722/2145] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/kika.py     | 115 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/kika.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..5ad4e9c36 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
+from .kika import KikaIE
 from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
new file mode 100644
index 000000000..db0f333ff
--- /dev/null
+++ b/youtube_dl/extractor/kika.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class KikaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|sendung)(?P<id>\d+).*'
+
+    _TESTS = [
+        {
+            'url': 'http://www.kika.de/baumhaus/videos/video9572.html',
+            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'info_dict': {
+                'id': '9572',
+                'ext': 'mp4',
+                'title': 'Baumhaus vom 29. Oktober 2014',
+                'description': None
+            }
+        },
+        {
+            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
+            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+            'info_dict': {
+                'id': '8182',
+                'ext': 'mp4',
+                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
+            }
+        },
+        {
+            'url': 'http://www.kika.de/videos/allevideos/video9572_zc-32ca94ad_zs-3f535991.html',
+            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'info_dict': {
+                'id': '9572',
+                'ext': 'mp4',
+                'title': 'Baumhaus vom 29. Oktober 2014',
+                'description': None
+            }
+        },
+        {
+            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/sendung81244_zc-81d703f8_zs-f82d5e31.html',
+            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+            'info_dict': {
+                'id': '8182',
+                'ext': 'mp4',
+                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        # broadcast_id may be the same as the video_id
+        broadcast_id = self._match_id(url)
+        webpage = self._download_webpage(url, broadcast_id)
+
+        xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
+        video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
+        if not video_id:
+            # Video is not available online
+            err_msg = 'Video %s is not available online' % broadcast_id
+            raise ExtractorError(err_msg, expected=True)
+
+        xml_url = 'http://www.kika.de/video%s-avCustom.xml' % (video_id)
+        xml_tree = self._download_xml(xml_url, video_id)
+
+        title = xml_tree.find('title').text
+        webpage_url = xml_tree.find('htmlUrl').text
+
+        # Try to get the description, not available for all videos
+        try:
+            broadcast_elem = xml_tree.find('broadcast')
+            description = broadcast_elem.find('broadcastDescription').text
+        except AttributeError:
+            # No description available
+            description = None
+
+        # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
+        tmp = xml_tree.find('duration').text.split(':')
+        duration = int(tmp[0]) * 60 + int(tmp[1])
+
+        formats_list = []
+        for elem in xml_tree.find('assets'):
+            format_dict = {}
+            format_dict['url'] = elem.find('progressiveDownloadUrl').text
+            format_dict['ext'] = elem.find('mediaType').text.lower()
+            format_dict['format'] = elem.find('profileName').text
+            width = int(elem.find('frameWidth').text)
+            height = int(elem.find('frameHeight').text)
+            format_dict['width'] = width
+            format_dict['height'] = height
+            format_dict['resolution'] = '%dx%d' % (width, height)
+            format_dict['abr'] = int(elem.find('bitrateAudio').text)
+            format_dict['vbr'] = int(elem.find('bitrateVideo').text)
+            format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
+            format_dict['filesize'] = int(elem.find('fileSize').text)
+
+            # append resolution and dict for sorting by resolution
+            formats_list.append((width * height, format_dict))
+
+        # Sort by resolution (=quality)
+        formats_list.sort()
+
+        out_list = [x[1] for x in formats_list]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': out_list,
+            'duration': duration,
+            'webpage_url': webpage_url
+        }

From 892015b088fa21915270b0a05937fcc7063ccdd2 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Mon, 28 Sep 2015 22:00:56 +0200
Subject: [PATCH 1723/2145] replaced inefficient code

---
 youtube_dl/extractor/kika.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index db0f333ff..871e4ea44 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -87,29 +87,25 @@ class KikaIE(InfoExtractor):
             format_dict['url'] = elem.find('progressiveDownloadUrl').text
             format_dict['ext'] = elem.find('mediaType').text.lower()
             format_dict['format'] = elem.find('profileName').text
-            width = int(elem.find('frameWidth').text)
-            height = int(elem.find('frameHeight').text)
-            format_dict['width'] = width
-            format_dict['height'] = height
-            format_dict['resolution'] = '%dx%d' % (width, height)
+            format_dict['width'] = int(elem.find('frameWidth').text)
+            format_dict['height'] = int(elem.find('frameHeight').text)
+            format_dict['resolution'] = '%dx%d' % (format_dict['width'],
+                                                   format_dict['height'])
             format_dict['abr'] = int(elem.find('bitrateAudio').text)
             format_dict['vbr'] = int(elem.find('bitrateVideo').text)
             format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
             format_dict['filesize'] = int(elem.find('fileSize').text)
 
-            # append resolution and dict for sorting by resolution
-            formats_list.append((width * height, format_dict))
+            formats_list.append(format_dict)
 
         # Sort by resolution (=quality)
-        formats_list.sort()
-
-        out_list = [x[1] for x in formats_list]
+        formats_list.sort(key=lambda x: x['width'] * x['height'])
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': out_list,
+            'formats': formats_list,
             'duration': duration,
             'webpage_url': webpage_url
         }

From 78d7ee19dc417b16b26fe2fa1101124866727a85 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 31 Oct 2015 22:21:52 +0800
Subject: [PATCH 1724/2145] [democracynow] Fix _TESTS

---
 youtube_dl/extractor/democracynow.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 973bb437b..05cfc7502 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -36,10 +36,9 @@ class DemocracynowIE(InfoExtractor):
         if display_id == '':
             display_id = 'home'
         webpage = self._download_webpage(url, display_id)
-        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1', webpage, re.DOTALL)
-        description = re_desc.group(2) if re_desc else ''
+        description = self._og_search_description(webpage)
 
-        jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
+        jstr = self._search_regex(r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json')
         js = self._parse_json(jstr, display_id)
         video_id = None
         formats = []
@@ -56,7 +55,7 @@ class DemocracynowIE(InfoExtractor):
                 'ext': ext,
                 'url': url,
             }]
-        for key in ('file', 'audio'):
+        for key in ('file', 'audio', 'video'):
             url = js.get(key, '')
             if url == '' or url is None:
                 continue

From 8c1aa28c27af204c4996260cdc70359e83c2c3d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Oct 2015 16:14:36 +0100
Subject: [PATCH 1725/2145] [kika] Replace non working tests and recognize
 'einzelsendung' urls.

---
 youtube_dl/extractor/kika.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index 871e4ea44..c9169076a 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -6,16 +6,16 @@ from ..utils import ExtractorError
 
 
 class KikaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|sendung)(?P<id>\d+).*'
+    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|(?:einzel)?sendung)(?P<id>\d+).*'
 
     _TESTS = [
         {
-            'url': 'http://www.kika.de/baumhaus/videos/video9572.html',
-            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
+            'md5': '4930515e36b06c111213e80d1e4aad0e',
             'info_dict': {
-                'id': '9572',
+                'id': '19636',
                 'ext': 'mp4',
-                'title': 'Baumhaus vom 29. Oktober 2014',
+                'title': 'Baumhaus vom 30. Oktober 2015',
                 'description': None
             }
         },
@@ -30,17 +30,17 @@ class KikaIE(InfoExtractor):
             }
         },
         {
-            'url': 'http://www.kika.de/videos/allevideos/video9572_zc-32ca94ad_zs-3f535991.html',
-            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
+            'md5': '4930515e36b06c111213e80d1e4aad0e',
             'info_dict': {
-                'id': '9572',
+                'id': '19636',
                 'ext': 'mp4',
-                'title': 'Baumhaus vom 29. Oktober 2014',
+                'title': 'Baumhaus vom 30. Oktober 2015',
                 'description': None
             }
         },
         {
-            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/sendung81244_zc-81d703f8_zs-f82d5e31.html',
+            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
             'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
             'info_dict': {
                 'id': '8182',

From c3040bd00a43e111dab0d1ab903df03ac7d19a00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Oct 2015 16:32:35 +0100
Subject: [PATCH 1726/2145] [kika] Cleanup

Closes #6957.
---
 youtube_dl/extractor/kika.py | 54 +++++++++++++++---------------------
 1 file changed, 22 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index c9169076a..5337ac439 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -16,8 +16,8 @@ class KikaIE(InfoExtractor):
                 'id': '19636',
                 'ext': 'mp4',
                 'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None
-            }
+                'description': None,
+            },
         },
         {
             'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
@@ -26,8 +26,8 @@ class KikaIE(InfoExtractor):
                 'id': '8182',
                 'ext': 'mp4',
                 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
-            }
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+            },
         },
         {
             'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
@@ -36,8 +36,8 @@ class KikaIE(InfoExtractor):
                 'id': '19636',
                 'ext': 'mp4',
                 'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None
-            }
+                'description': None,
+            },
         },
         {
             'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
@@ -46,9 +46,9 @@ class KikaIE(InfoExtractor):
                 'id': '8182',
                 'ext': 'mp4',
                 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
-            }
-        }
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -59,7 +59,6 @@ class KikaIE(InfoExtractor):
         xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
         video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
         if not video_id:
-            # Video is not available online
             err_msg = 'Video %s is not available online' % broadcast_id
             raise ExtractorError(err_msg, expected=True)
 
@@ -74,38 +73,29 @@ class KikaIE(InfoExtractor):
             broadcast_elem = xml_tree.find('broadcast')
             description = broadcast_elem.find('broadcastDescription').text
         except AttributeError:
-            # No description available
             description = None
 
         # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
         tmp = xml_tree.find('duration').text.split(':')
         duration = int(tmp[0]) * 60 + int(tmp[1])
 
-        formats_list = []
-        for elem in xml_tree.find('assets'):
-            format_dict = {}
-            format_dict['url'] = elem.find('progressiveDownloadUrl').text
-            format_dict['ext'] = elem.find('mediaType').text.lower()
-            format_dict['format'] = elem.find('profileName').text
-            format_dict['width'] = int(elem.find('frameWidth').text)
-            format_dict['height'] = int(elem.find('frameHeight').text)
-            format_dict['resolution'] = '%dx%d' % (format_dict['width'],
-                                                   format_dict['height'])
-            format_dict['abr'] = int(elem.find('bitrateAudio').text)
-            format_dict['vbr'] = int(elem.find('bitrateVideo').text)
-            format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
-            format_dict['filesize'] = int(elem.find('fileSize').text)
-
-            formats_list.append(format_dict)
-
-        # Sort by resolution (=quality)
-        formats_list.sort(key=lambda x: x['width'] * x['height'])
+        formats = [{
+            'url': elem.find('progressiveDownloadUrl').text,
+            'ext': elem.find('mediaType').text.lower(),
+            'format': elem.find('profileName').text,
+            'width': int(elem.find('frameWidth').text),
+            'height': int(elem.find('frameHeight').text),
+            'abr': int(elem.find('bitrateAudio').text),
+            'vbr': int(elem.find('bitrateVideo').text),
+            'filesize': int(elem.find('fileSize').text),
+        } for elem in xml_tree.find('assets')]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': formats_list,
+            'formats': formats,
             'duration': duration,
-            'webpage_url': webpage_url
+            'webpage_url': webpage_url,
         }

From 2b1b2d83cacfdce19cae5eea2f9bbfd142efc7f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:17:09 +0600
Subject: [PATCH 1727/2145] [mdr] Modernize and include kika.de

---
 youtube_dl/extractor/__init__.py |   1 -
 youtube_dl/extractor/kika.py     | 101 ------------------
 youtube_dl/extractor/mdr.py      | 172 +++++++++++++++++++++++--------
 3 files changed, 131 insertions(+), 143 deletions(-)
 delete mode 100644 youtube_dl/extractor/kika.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5ad4e9c36..f98e6487e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,6 @@ from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
-from .kika import KikaIE
 from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
deleted file mode 100644
index 5337ac439..000000000
--- a/youtube_dl/extractor/kika.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class KikaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|(?:einzel)?sendung)(?P<id>\d+).*'
-
-    _TESTS = [
-        {
-            'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
-            'md5': '4930515e36b06c111213e80d1e4aad0e',
-            'info_dict': {
-                'id': '19636',
-                'ext': 'mp4',
-                'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None,
-            },
-        },
-        {
-            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
-            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
-            'info_dict': {
-                'id': '8182',
-                'ext': 'mp4',
-                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
-            },
-        },
-        {
-            'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
-            'md5': '4930515e36b06c111213e80d1e4aad0e',
-            'info_dict': {
-                'id': '19636',
-                'ext': 'mp4',
-                'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None,
-            },
-        },
-        {
-            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
-            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
-            'info_dict': {
-                'id': '8182',
-                'ext': 'mp4',
-                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
-            },
-        },
-    ]
-
-    def _real_extract(self, url):
-        # broadcast_id may be the same as the video_id
-        broadcast_id = self._match_id(url)
-        webpage = self._download_webpage(url, broadcast_id)
-
-        xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
-        video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
-        if not video_id:
-            err_msg = 'Video %s is not available online' % broadcast_id
-            raise ExtractorError(err_msg, expected=True)
-
-        xml_url = 'http://www.kika.de/video%s-avCustom.xml' % (video_id)
-        xml_tree = self._download_xml(xml_url, video_id)
-
-        title = xml_tree.find('title').text
-        webpage_url = xml_tree.find('htmlUrl').text
-
-        # Try to get the description, not available for all videos
-        try:
-            broadcast_elem = xml_tree.find('broadcast')
-            description = broadcast_elem.find('broadcastDescription').text
-        except AttributeError:
-            description = None
-
-        # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
-        tmp = xml_tree.find('duration').text.split(':')
-        duration = int(tmp[0]) * 60 + int(tmp[1])
-
-        formats = [{
-            'url': elem.find('progressiveDownloadUrl').text,
-            'ext': elem.find('mediaType').text.lower(),
-            'format': elem.find('profileName').text,
-            'width': int(elem.find('frameWidth').text),
-            'height': int(elem.find('frameHeight').text),
-            'abr': int(elem.find('bitrateAudio').text),
-            'vbr': int(elem.find('bitrateVideo').text),
-            'filesize': int(elem.find('fileSize').text),
-        } for elem in xml_tree.find('assets')]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'formats': formats,
-            'duration': duration,
-            'webpage_url': webpage_url,
-        }
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index fc7499958..541ddd909 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -1,64 +1,154 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    xpath_text,
+)
 
 
 class MDRIE(InfoExtractor):
-    _VALID_URL = r'^(?P<domain>https?://(?:www\.)?mdr\.de)/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)(?:_|\.html)'
+    IE_DESC = 'MDR.DE and KiKA'
+    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
 
-    # No tests, MDR regularily deletes its videos
-    _TEST = {
+    _TESTS = [{
+        # MDR regularily deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
-    }
+    }, {
+        'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
+        'md5': '4930515e36b06c111213e80d1e4aad0e',
+        'info_dict': {
+            'id': '19636',
+            'ext': 'mp4',
+            'title': 'Baumhaus vom 30. Oktober 2015',
+            'duration': 134,
+            'uploader': 'KIKA',
+        },
+    }, {
+        'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
+        'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+        'info_dict': {
+            'id': '8182',
+            'ext': 'mp4',
+            'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+            'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+            'timestamp': 1419047100,
+            'upload_date': '20141220',
+            'duration': 4628,
+            'uploader': 'KIKA',
+        },
+    }, {
+        'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('video_id')
-        domain = m.group('domain')
+        video_id = self._match_id(url)
 
-        # determine title and media streams from webpage
-        html = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h[12]>(.*?)</h[12]>', html, 'title')
-        xmlurl = self._search_regex(
-            r'dataURL:\'(/(?:.+)/(?:video|audio)[0-9]+-avCustom.xml)', html, 'XML URL')
+        data_url = self._search_regex(
+            r'dataURL\s*:\s*(["\'])(?P<url>/.+/(?:video|audio)[0-9]+-avCustom\.xml)\1',
+            webpage, 'data url', group='url')
+
+        doc = self._download_xml(
+            compat_urlparse.urljoin(url, data_url), video_id)
+
+        title = (xpath_text(doc, './title', 'title', default=None) or
+                 xpath_text(doc, './broadcast/broadcastName', 'title'))
 
-        doc = self._download_xml(domain + xmlurl, video_id)
         formats = []
-        for a in doc.findall('./assets/asset'):
-            url_el = a.find('./progressiveDownloadUrl')
-            if url_el is None:
-                continue
-            abr = int(a.find('bitrateAudio').text) // 1000
-            media_type = a.find('mediaType').text
-            format = {
-                'abr': abr,
-                'filesize': int(a.find('fileSize').text),
-                'url': url_el.text,
-            }
+        processed_urls = []
+        for asset in doc.findall('./assets/asset'):
+            for source in (
+                    'progressiveDownload',
+                    'dynamicHttpStreamingRedirector',
+                    'adaptiveHttpStreamingRedirector'):
+                url_el = asset.find('./%sUrl' % source)
+                if url_el is None:
+                    continue
 
-            vbr_el = a.find('bitrateVideo')
-            if vbr_el is None:
-                format.update({
-                    'vcodec': 'none',
-                    'format_id': '%s-%d' % (media_type, abr),
-                })
-            else:
-                vbr = int(vbr_el.text) // 1000
-                format.update({
-                    'vbr': vbr,
-                    'width': int(a.find('frameWidth').text),
-                    'height': int(a.find('frameHeight').text),
-                    'format_id': '%s-%d' % (media_type, vbr),
-                })
-            formats.append(format)
+                video_url = url_el.text
+                if video_url in processed_urls:
+                    continue
+
+                processed_urls.append(video_url)
+
+                vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
+                abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
+
+                url_formats = []
+
+                ext = determine_ext(url_el.text)
+                if ext == 'm3u8':
+                    url_formats = self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        preference=0, m3u8_id='HLS', fatal=False)
+                elif ext == 'f4m':
+                    url_formats = self._extract_f4m_formats(
+                        video_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
+                        preference=0, f4m_id='HDS', fatal=False)
+                else:
+                    media_type = xpath_text(asset, './mediaType', 'media type', default='MP4')
+                    vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
+                    abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
+                    filesize = int_or_none(xpath_text(asset, './fileSize', 'file size'))
+
+                    f = {
+                        'url': video_url,
+                        'format_id': '%s-%d' % (media_type, vbr or abr),
+                        'filesize': filesize,
+                        'abr': abr,
+                        'preference': 1,
+                    }
+
+                    if vbr:
+                        width = int_or_none(xpath_text(asset, './frameWidth', 'width'))
+                        height = int_or_none(xpath_text(asset, './frameHeight', 'height'))
+                        f.update({
+                            'vbr': vbr,
+                            'width': width,
+                            'height': height,
+                        })
+
+                    url_formats.append(f)
+
+                if not vbr:
+                    for f in url_formats:
+                        abr = f.get('tbr') or abr
+                        if 'tbr' in f:
+                            del f['tbr']
+                        f.update({
+                            'abr': abr,
+                            'vcodec': 'none',
+                        })
+
+                if url_formats:
+                    formats.extend(url_formats)
         self._sort_formats(formats)
 
+        description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
+        timestamp = parse_iso8601(
+            xpath_text(doc, './broadcast/broadcastDate', 'timestamp', default=None) or
+            xpath_text(doc, './broadcast/broadcastStartDate', 'timestamp', default=None))
+        duration = parse_duration(xpath_text(doc, './duration', 'duration'))
+        uploader = xpath_text(doc, './rights', 'uploader')
+
         return {
             'id': video_id,
             'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'duration': duration,
+            'uploader': uploader,
             'formats': formats,
         }

From 8cdb5c845336ad3dc48c85a0558a38bd42972b00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:24:21 +0600
Subject: [PATCH 1728/2145] [mdr] Add audio test

---
 youtube_dl/extractor/mdr.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 541ddd909..e05577496 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -20,6 +20,17 @@ class MDRIE(InfoExtractor):
         # MDR regularily deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
+    },  {
+        # audio
+        'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
+        'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',
+        'info_dict': {
+            'id': '1312272',
+            'ext': 'mp3',
+            'title': 'Feuilleton vom 30. Oktober 2015',
+            'duration': 250,
+            'uploader': 'MITTELDEUTSCHER RUNDFUNK',
+        },
     }, {
         'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
         'md5': '4930515e36b06c111213e80d1e4aad0e',

From 578c074575f45ffdfd032d7b84f6fe449614f511 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:39:44 +0600
Subject: [PATCH 1729/2145] [utils] Support list of xpath in xpath_element

---
 test/test_utils.py  |  7 +++++++
 youtube_dl/utils.py | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 0c34f0e55..5a56ad776 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -275,9 +275,16 @@ class TestUtil(unittest.TestCase):
         p = xml.etree.ElementTree.SubElement(div, 'p')
         p.text = 'Foo'
         self.assertEqual(xpath_element(doc, 'div/p'), p)
+        self.assertEqual(xpath_element(doc, ['div/p']), p)
+        self.assertEqual(xpath_element(doc, ['div/bar', 'div/p']), p)
         self.assertEqual(xpath_element(doc, 'div/bar', default='default'), 'default')
+        self.assertEqual(xpath_element(doc, ['div/bar'], default='default'), 'default')
         self.assertTrue(xpath_element(doc, 'div/bar') is None)
+        self.assertTrue(xpath_element(doc, ['div/bar']) is None)
+        self.assertTrue(xpath_element(doc, ['div/bar'], 'div/baz') is None)
         self.assertRaises(ExtractorError, xpath_element, doc, 'div/bar', fatal=True)
+        self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar'], fatal=True)
+        self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar', 'div/baz'], fatal=True)
 
     def test_xpath_text(self):
         testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 558c9c7d5..89c88a4d3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -178,10 +178,19 @@ def xpath_with_ns(path, ns_map):
 
 
 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
-    if sys.version_info < (2, 7):  # Crazy 2.6
-        xpath = xpath.encode('ascii')
+    def _find_xpath(xpath):
+        if sys.version_info < (2, 7):  # Crazy 2.6
+            xpath = xpath.encode('ascii')
+        return node.find(xpath)
+
+    if isinstance(xpath, (str, compat_str)):
+        n = _find_xpath(xpath)
+    else:
+        for xp in xpath:
+            n = _find_xpath(xp)
+            if n is not None:
+                break
 
-    n = node.find(xpath)
     if n is None:
         if default is not NO_DEFAULT:
             return default

From 11465da70257663ee52c7be50debe1c1e825ec67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:45:45 +0600
Subject: [PATCH 1730/2145] [mdr] Simplify xpath

---
 youtube_dl/extractor/mdr.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index e05577496..a63257c56 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -74,8 +74,7 @@ class MDRIE(InfoExtractor):
         doc = self._download_xml(
             compat_urlparse.urljoin(url, data_url), video_id)
 
-        title = (xpath_text(doc, './title', 'title', default=None) or
-                 xpath_text(doc, './broadcast/broadcastName', 'title'))
+        title = xpath_text(doc, ['./title', './broadcast/broadcastName'], 'title', fatal=True)
 
         formats = []
         processed_urls = []
@@ -149,8 +148,12 @@ class MDRIE(InfoExtractor):
 
         description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
         timestamp = parse_iso8601(
-            xpath_text(doc, './broadcast/broadcastDate', 'timestamp', default=None) or
-            xpath_text(doc, './broadcast/broadcastStartDate', 'timestamp', default=None))
+            xpath_text(
+                doc, [
+                    './broadcast/broadcastDate',
+                    './broadcast/broadcastStartDate',
+                    './broadcast/broadcastEndDate'],
+                'timestamp', default=None))
         duration = parse_duration(xpath_text(doc, './duration', 'duration'))
         uploader = xpath_text(doc, './rights', 'uploader')
 

From 82b69a5cbb1455d31916be4f19ab327ae63f313c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:00:36 +0600
Subject: [PATCH 1731/2145] [mdr] PEP 8

---
 youtube_dl/extractor/mdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index a63257c56..a566c6a2c 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -20,7 +20,7 @@ class MDRIE(InfoExtractor):
         # MDR regularily deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
-    },  {
+    }, {
         # audio
         'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
         'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',

From e327b736ca6a6a1c880b93e09a3b310c354c2c7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:05:30 +0600
Subject: [PATCH 1732/2145] [generic] Update test

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..a84135032 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -141,6 +141,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Automatics, robotics and biocybernetics',
                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+                'upload_date': '20130627',
                 'formats': 'mincount:16',
                 'subtitles': 'mincount:1',
             },

From ae12bc3ebb4cb377c2b4337ec255e652b36f5143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:07:37 +0600
Subject: [PATCH 1733/2145] [utils] Make unified_strdate always return unicode
 string

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 89c88a4d3..764a89cca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -910,7 +910,7 @@ def unified_strdate(date_str, day_first=True):
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
-    return upload_date
+    return compat_str(upload_date)
 
 
 def determine_ext(url, default_ext='unknown_video'):

From dc519b5421366a8cac681455a817ae25f7f4aa83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:12:57 +0600
Subject: [PATCH 1734/2145] [extractor/common] Make ie_key and IE_NAME return
 unicode string

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 10c0d5d1f..59c3fa8dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -310,11 +310,11 @@ class InfoExtractor(object):
     @classmethod
     def ie_key(cls):
         """A string for getting the InfoExtractor with get_info_extractor"""
-        return cls.__name__[:-2]
+        return compat_str(cls.__name__[:-2])
 
     @property
     def IE_NAME(self):
-        return type(self).__name__[:-2]
+        return compat_str(type(self).__name__[:-2])
 
     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
         """ Returns the response handle """

From 76f0c50d3d3e2eb5903b61da08829699e902916d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 00:01:08 +0600
Subject: [PATCH 1735/2145] [mdr] Fix failed formats processing

---
 youtube_dl/extractor/mdr.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index a566c6a2c..88334889e 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -96,8 +96,6 @@ class MDRIE(InfoExtractor):
                 vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
                 abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
 
-                url_formats = []
-
                 ext = determine_ext(url_el.text)
                 if ext == 'm3u8':
                     url_formats = self._extract_m3u8_formats(
@@ -130,7 +128,10 @@ class MDRIE(InfoExtractor):
                             'height': height,
                         })
 
-                    url_formats.append(f)
+                    url_formats = [f]
+
+                if not url_formats:
+                    continue
 
                 if not vbr:
                     for f in url_formats:
@@ -142,8 +143,8 @@ class MDRIE(InfoExtractor):
                             'vcodec': 'none',
                         })
 
-                if url_formats:
-                    formats.extend(url_formats)
+                formats.extend(url_formats)
+
         self._sort_formats(formats)
 
         description = xpath_text(doc, './broadcast/broadcastDescription', 'description')

From dbd82a1d4fff1655920e111cc25a7fd526d7bf9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 00:01:34 +0600
Subject: [PATCH 1736/2145] [extractor/common] Fix m3u8 extraction on failure

---
 youtube_dl/extractor/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 59c3fa8dc..1f09fbb47 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -943,13 +943,14 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        m3u8_doc, urlh = self._download_webpage_handle(
+        res = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
-        if m3u8_doc is False:
-            return m3u8_doc
+        if res is False:
+            return res
+        m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
         last_info = None
         last_media = None

From 9550ca506fccf9c9d795816cc0a7817ff262ef45 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 19:36:04 +0100
Subject: [PATCH 1737/2145] [utils] change extract_attributes to work in python
 2

---
 youtube_dl/extractor/brightcove.py | 3 +--
 youtube_dl/utils.py                | 3 ++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b41cee91b..c6ad1d065 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -383,8 +383,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
         return None
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        account_id, player_id, embed, video_id = mobj.groups()
+        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bcebf9cc5..518cea98b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -252,7 +252,8 @@ def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s
     attributes = re.findall(attributes_regex, attributes_str)
     attributes_dict = {}
     if attributes:
-        attributes_dict = {attribute_name: attribute_value for (attribute_name, attribute_value) in attributes}
+        for (attribute_name, attribute_value) in attributes:
+            attributes_dict[attribute_name] = attribute_value
     return attributes_dict
 
 

From 80dcee5cd5cbe623a53e0c582e3e3ae170c63e8d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 04:02:49 +0100
Subject: [PATCH 1738/2145] [eitb] fix info extraction

---
 youtube_dl/extractor/eitb.py | 65 ++++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 2cba82532..fc8f15544 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -1,39 +1,62 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
-from ..utils import ExtractorError
+from ..compat import compat_urllib_request
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 
 
 class EitbIE(InfoExtractor):
     IE_NAME = 'eitb.tv'
-    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
+    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
 
     _TEST = {
-        'add_ie': ['Brightcove'],
-        'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
+        'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
         'md5': 'edf4436247185adee3ea18ce64c47998',
         'info_dict': {
-            'id': '2743577154001',
+            'id': '4090227752001',
             'ext': 'mp4',
             'title': '60 minutos (Lasa y Zabala, 30 años)',
-            # All videos from eitb has this description in the brightcove info
-            'description': '.',
-            'uploader': 'Euskal Telebista',
+            'description': '',
+            'duration': 3996760,
+            'upload_date': '20131014',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        chapter_id = mobj.group('chapter_id')
-        webpage = self._download_webpage(url, chapter_id)
-        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
-        if bc_url is None:
-            raise ExtractorError('Could not extract the Brightcove url')
-        # The BrightcoveExperience object doesn't contain the video id, we set
-        # it manually
-        bc_url += '&%40videoPlayer={0}'.format(chapter_id)
-        return self.url_result(bc_url, BrightcoveIE.ie_key())
+        video_id = self._match_id(url)
+        video_data = self._download_json('http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id, video_id)['web_media'][0]
+
+        formats = []
+        for rendition in video_data['RENDITIONS']:
+            formats.append({
+                'url': rendition['PMD_URL'],
+                'width': int_or_none(rendition.get('FRAME_WIDTH')),
+                'height': int_or_none(rendition.get('FRAME_HEIGHT')),
+                'tbr': int_or_none(rendition.get('ENCODING_RATE')),
+            })
+
+        # TODO: parse f4m manifest
+        request = compat_urllib_request.Request(
+            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
+            headers={'Referer': url})
+        token_data = self._download_json(request, video_id, fatal=False)
+        if token_data:
+            m3u8_formats = self._extract_m3u8_formats('%s?hdnts=%s' % (video_data['HLS_SURL'], token_data['token']), video_id, m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data['NAME_ES'],
+            'description': video_data.get('SHORT_DESC_ES'),
+            'thumbnail': video_data.get('STILL_URL'),
+            'duration': int_or_none(video_data.get('LENGTH')),
+            'upload_date': unified_strdate(video_data.get('BROADCST_DATE')),
+            'formats': formats,
+        }

From 8a06999ba0f9c948f8d2a1ef89c73eedbfb09cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 01:52:33 +0600
Subject: [PATCH 1739/2145] [eitb] Improve, make more robust and extract f4m
 formats (Closes #7328)

---
 youtube_dl/extractor/eitb.py | 71 +++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index fc8f15544..0de8d3dc6 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -4,14 +4,15 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import (
+    float_or_none,
     int_or_none,
-    unified_strdate,
+    parse_iso8601,
 )
 
 
 class EitbIE(InfoExtractor):
     IE_NAME = 'eitb.tv'
-    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?eitb\.tv/(?:eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
@@ -20,43 +21,71 @@ class EitbIE(InfoExtractor):
             'id': '4090227752001',
             'ext': 'mp4',
             'title': '60 minutos (Lasa y Zabala, 30 años)',
-            'description': '',
-            'duration': 3996760,
+            'description': 'Programa de reportajes de actualidad.',
+            'duration': 3996.76,
+            'timestamp': 1381789200,
             'upload_date': '20131014',
+            'tags': list,
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json('http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id, video_id)['web_media'][0]
+
+        video = self._download_json(
+            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id,
+            video_id, 'Downloading video JSON')
+
+        media = video['web_media'][0]
 
         formats = []
-        for rendition in video_data['RENDITIONS']:
+        for rendition in media['RENDITIONS']:
+            video_url = rendition.get('PMD_URL')
+            if not video_url:
+                continue
+            tbr = float_or_none(rendition.get('ENCODING_RATE'), 1000)
+            format_id = 'http'
+            if tbr:
+                format_id += '-%d' % int(tbr)
             formats.append({
                 'url': rendition['PMD_URL'],
+                'format_id': format_id,
                 'width': int_or_none(rendition.get('FRAME_WIDTH')),
                 'height': int_or_none(rendition.get('FRAME_HEIGHT')),
-                'tbr': int_or_none(rendition.get('ENCODING_RATE')),
+                'tbr': tbr,
             })
 
-        # TODO: parse f4m manifest
-        request = compat_urllib_request.Request(
-            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
-            headers={'Referer': url})
-        token_data = self._download_json(request, video_id, fatal=False)
-        if token_data:
-            m3u8_formats = self._extract_m3u8_formats('%s?hdnts=%s' % (video_data['HLS_SURL'], token_data['token']), video_id, m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+        hls_url = media.get('HLS_SURL')
+        if hls_url:
+            request = compat_urllib_request.Request(
+                'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
+                headers={'Referer': url})
+            token_data = self._download_json(
+                request, video_id, 'Downloading auth token', fatal=False)
+            if token_data:
+                token = token_data.get('token')
+                if token:
+                    m3u8_formats = self._extract_m3u8_formats(
+                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+
+        hds_url = media.get('HDS_SURL').replace('euskalsvod', 'euskalvod')
+        if hds_url:
+            f4m_formats = self._extract_f4m_formats(
+                '%s?hdcore=3.7.0' % hds_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
 
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video_data['NAME_ES'],
-            'description': video_data.get('SHORT_DESC_ES'),
-            'thumbnail': video_data.get('STILL_URL'),
-            'duration': int_or_none(video_data.get('LENGTH')),
-            'upload_date': unified_strdate(video_data.get('BROADCST_DATE')),
+            'title': media.get('NAME_ES') or media.get('name') or media['NAME_EU'],
+            'description': media.get('SHORT_DESC_ES') or video.get('desc_group') or media.get('SHORT_DESC_EU'),
+            'thumbnail': media.get('STILL_URL') or media.get('THUMBNAIL_URL'),
+            'duration': float_or_none(media.get('LENGTH'), 1000),
+            'timestamp': parse_iso8601(media.get('BROADCST_DATE'), ' '),
+            'tags': media.get('TAGS'),
             'formats': formats,
         }

From 999079b4543b4cd5e71a235865fbfefd349eb064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 15:49:11 +0600
Subject: [PATCH 1740/2145] [eitb] Improve hds extraction

---
 youtube_dl/extractor/eitb.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 0de8d3dc6..357a2196c 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -70,10 +70,11 @@ class EitbIE(InfoExtractor):
                     if m3u8_formats:
                         formats.extend(m3u8_formats)
 
-        hds_url = media.get('HDS_SURL').replace('euskalsvod', 'euskalvod')
+        hds_url = media.get('HDS_SURL')
         if hds_url:
             f4m_formats = self._extract_f4m_formats(
-                '%s?hdcore=3.7.0' % hds_url, video_id, f4m_id='hds', fatal=False)
+                '%s?hdcore=3.7.0' % hds_url.replace('euskalsvod', 'euskalvod'),
+                video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
                 formats.extend(f4m_formats)
 

From ab6ca0480280abb2a35a54e1b380bbae07a48863 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Nov 2015 14:20:10 +0100
Subject: [PATCH 1741/2145] release 2015.11.01

---
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 03561b87d..805af14a0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -93,6 +93,7 @@
  - **Clipsyndicate**
  - **Cloudy**
  - **Clubic**
+ - **Clyp**
  - **cmt.com**
  - **CNET**
  - **CNN**
@@ -281,7 +282,7 @@
  - **macgamestore**: MacGameStore trailers
  - **mailru**: Видео@Mail.Ru
  - **Malemotion**
- - **MDR**
+ - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **metacafe**
  - **Metacritic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 125e8ccf5..006b973c0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.24'
+__version__ = '2015.11.01'

From c90d16cf36d8edf03f4dc923ee9dbeadca910844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Nov 2015 04:26:20 +0600
Subject: [PATCH 1742/2145] [utils:sanitize_path] Disallow trailing whitespace
 in path segment (Closes #7332)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index efd5f4ae1..7b3f79141 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -366,7 +366,7 @@ def sanitize_path(s):
     if drive_or_unc:
         norm_path.pop(0)
     sanitized_path = [
-        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
+        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
         for path_part in norm_path]
     if drive_or_unc:
         sanitized_path.insert(0, drive_or_unc + os.path.sep)

From eb97f46e8bd9cb04f0fe5f8a5c13aeeaabeefef5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Nov 2015 12:46:10 +0100
Subject: [PATCH 1743/2145] [mitele] Fix extraction and update test checksum
 (fixes #7343)

---
 youtube_dl/extractor/mitele.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 3142fcde2..c595f2077 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     encode_dict,
     get_element_by_attribute,
@@ -15,7 +18,7 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': '757b0b66cbd7e0a97226d7d3156cb3e9',
+        'md5': '0ff1a13aebb35d9bc14081ff633dd324',
         'info_dict': {
             'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',
@@ -34,6 +37,7 @@ class MiTeleIE(InfoExtractor):
 
         config_url = self._search_regex(
             r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url')
+        config_url = compat_urlparse.urljoin(url, config_url)
 
         config = self._download_json(
             config_url, display_id, 'Downloading config JSON')

From c514b0ec655b23e7804eb18df04daa863d973f32 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 22:12:20 +0100
Subject: [PATCH 1744/2145] [videofy.me] fix info extraction

Closes #7339.
---
 youtube_dl/extractor/videofyme.py | 40 ++++++++++++++++---------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index 94f9e9be9..cd3f50a63 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -2,8 +2,8 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    find_xpath_attr,
     int_or_none,
+    parse_iso8601,
 )
 
 
@@ -18,33 +18,35 @@ class VideofyMeIE(InfoExtractor):
             'id': '1100701',
             'ext': 'mp4',
             'title': 'This is VideofyMe',
-            'description': None,
+            'description': '',
+            'upload_date': '20130326',
+            'timestamp': 1364288959,
             'uploader': 'VideofyMe',
             'uploader_id': 'thisisvideofyme',
             'view_count': int,
+            'likes': int,
+            'comment_count': int,
         },
-
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        config = self._download_xml('http://sunshine.videofy.me/?videoId=%s' % video_id,
-                                    video_id)
-        video = config.find('video')
-        sources = video.find('sources')
-        url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
-                                          for key in ['on', 'av', 'off']] if node is not None)
-        video_url = url_node.find('url').text
-        view_count = int_or_none(self._search_regex(
-            r'([0-9]+)', video.find('views').text, 'view count', fatal=False))
+
+        config = self._download_json('http://vf-player-info-loader.herokuapp.com/%s.json' % video_id, video_id)['videoinfo']
+
+        video = config.get('video')
+        blog = config.get('blog', {})
 
         return {
             'id': video_id,
-            'title': video.find('title').text,
-            'url': video_url,
-            'thumbnail': video.find('thumb').text,
-            'description': video.find('description').text,
-            'uploader': config.find('blog/name').text,
-            'uploader_id': video.find('identifier').text,
-            'view_count': view_count,
+            'title': video['title'],
+            'url': video['sources']['source']['url'],
+            'thumbnail': video.get('thumb'),
+            'description': video.get('description'),
+            'timestamp': parse_iso8601(video.get('date')),
+            'uploader': blog.get('name'),
+            'uploader_id': blog.get('identifier'),
+            'view_count': int_or_none(self._search_regex(r'([0-9]+)', video.get('views'), 'view count', fatal=False)),
+            'likes': int_or_none(video.get('likes')),
+            'comment_count': int_or_none(video.get('nrOfComments')),
         }

From 6a750402787dfc1f39a9ad347f2d78ae1c94c52c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Nov 2015 14:08:38 +0100
Subject: [PATCH 1745/2145] [utils] unified_strdate: Return None if the date
 format can't be recognized (fixes #7340)

This issue was introduced with ae12bc3ebb4cb377c2b4337ec255e652b36f5143, it returned 'None'.
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3298315d2..01829f71e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -236,6 +236,7 @@ class TestUtil(unittest.TestCase):
             unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
             '20150202')
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
+        self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7b3f79141..d39f313a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -911,7 +911,8 @@ def unified_strdate(date_str, day_first=True):
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
-    return compat_str(upload_date)
+    if upload_date is not None:
+        return compat_str(upload_date)
 
 
 def determine_ext(url, default_ext='unknown_video'):

From a230068ff7427c19e29331fc0f2bb17d50003bca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Nov 2015 16:18:54 +0100
Subject: [PATCH 1746/2145] release 2015.11.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 006b973c0..6ef482b78 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.01'
+__version__ = '2015.11.02'

From dde9fe9788f23f168e0bddaf8ab0470f469165fa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Nov 2015 18:36:54 +0800
Subject: [PATCH 1747/2145] [democracynow] Simplify

---
 youtube_dl/extractor/democracynow.py | 86 ++++++++++++++--------------
 1 file changed, 44 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 05cfc7502..824b8e2c5 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -2,11 +2,18 @@
 from __future__ import unicode_literals
 
 import re
+import os.path
+
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    url_basename,
+    remove_start,
+)
 
 
 class DemocracynowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P<id>[^\?]*)'
+    _VALID_URL = r'https?://(?:www\.)?democracynow.org/(?P<id>[^\?]*)'
     IE_NAME = 'democracynow'
     _TESTS = [{
         'url': 'http://www.democracynow.org/shows/2015/7/3',
@@ -14,9 +21,7 @@ class DemocracynowIE(InfoExtractor):
             'id': '2015-0703-001',
             'ext': 'mp4',
             'title': 'July 03, 2015 - Democracy Now!',
-            'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
-            'uploader': 'Democracy Now',
-            'upload_date': None,
+            'description': 'A daily independent global news hour with Amy Goodman & Juan González "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
@@ -25,60 +30,57 @@ class DemocracynowIE(InfoExtractor):
             'ext': 'mp4',
             'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
             'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
-            'uploader': 'Democracy Now',
-            'upload_date': None,
         },
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        base_host = re.search(r'^(.+?://[^/]+)', url).group(1)
-        if display_id == '':
-            display_id = 'home'
         webpage = self._download_webpage(url, display_id)
         description = self._og_search_description(webpage)
 
-        jstr = self._search_regex(r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json')
-        js = self._parse_json(jstr, display_id)
+        js = self._parse_json(self._search_regex(
+            r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
+            display_id)
         video_id = None
         formats = []
+
+        default_lang = 'en'
+
         subtitles = {}
-        for key in ('caption_file', '.......'):
-            # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
-            url = js.get(key, '')
-            if url == '' or url is None:
-                continue
-            if not re.match(r'^https?://', url):
-                url = base_host + url
-            ext = re.search(r'\.([^\.]+)$', url).group(1)
-            subtitles['eng'] = [{
-                'ext': ext,
-                'url': url,
-            }]
-        for key in ('file', 'audio', 'video'):
-            url = js.get(key, '')
-            if url == '' or url is None:
-                continue
-            if not re.match(r'^https?://', url):
-                url = base_host + url
-            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)', url)
-            if video_id is None:
-                video_id = purl.group('fn')
-            if js.get('start') is not None:
-                url += '&' if purl.group('hasparams') == '?' else '?'
-                url = url + 'start=' + str(js.get('start'))
-            formats.append({
-                'format_id': purl.group('dir'),
-                'ext': purl.group('ext'),
-                'url': url,
+
+        def add_subtitle_item(lang, info_dict):
+            if lang not in subtitles:
+                subtitles[lang] = []
+            subtitles[lang].append(info_dict)
+
+        # chapter_file are not subtitles
+        if 'caption_file' in js:
+            add_subtitle_item(default_lang, {
+                'url': compat_urlparse.urljoin(url, js['caption_file']),
             })
+
+        for subtitle_item in js.get('captions', []):
+            lang = subtitle_item.get('language', '').lower() or default_lang
+            add_subtitle_item(lang, {
+                'url': compat_urlparse.urljoin(url, subtitle_item['url']),
+            })
+
+        for key in ('file', 'audio', 'video'):
+            media_url = js.get(key, '')
+            if not media_url:
+                continue
+            media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
+            video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn')
+            formats.append({
+                'url': media_url,
+            })
+
         self._sort_formats(formats)
-        ret = {
+
+        return {
             'id': video_id,
             'title': js.get('title'),
             'description': description,
-            'uploader': 'Democracy Now',
             'subtitles': subtitles,
             'formats': formats,
         }
-        return ret

From fc68d52bb95dc81ed3d05a5c5397cd3f35ee093a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Nov 2015 21:24:10 +0800
Subject: [PATCH 1748/2145] [democracynow] Add MD5 sums

---
 youtube_dl/extractor/democracynow.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 824b8e2c5..70c364e8b 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -17,6 +17,7 @@ class DemocracynowIE(InfoExtractor):
     IE_NAME = 'democracynow'
     _TESTS = [{
         'url': 'http://www.democracynow.org/shows/2015/7/3',
+        'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',
@@ -25,6 +26,7 @@ class DemocracynowIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
+        'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',

From 852fad922ffa931b3c90b0b9fdb2fa1c7f965ab4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 20:53:17 +0600
Subject: [PATCH 1749/2145] [vimeo] Fix non-ASCII video passwords (Closes
 #7352)

---
 youtube_dl/extractor/vimeo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2437ae1eb..cc0d337e8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
@@ -208,10 +209,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata({
+        data = urlencode_postdata(encode_dict({
             'password': password,
             'token': token,
-        })
+        }))
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')

From 0a0110fc6bbd21850e25541fd0bd4b602ce194e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:01:09 +0600
Subject: [PATCH 1750/2145] [vimeo] Fix non-ASCII video passwords (2)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cc0d337e8..fa07bd59c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -228,7 +228,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
-        data = compat_urllib_parse.urlencode({'password': password})
+        data = urlencode_postdata(encode_dict({'password': password}))
         pass_url = url + '/check-password'
         password_request = compat_urllib_request.Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')

From 3fa3ff1bc36aaf82ac0a5e880304cb7aae217b9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:06:36 +0600
Subject: [PATCH 1751/2145] [vimeo] Fix non-ASCII login

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fa07bd59c..46fb36f21 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -41,13 +41,13 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata({
+        data = urlencode_postdata(encode_dict({
             'action': 'login',
             'email': username,
             'password': password,
             'service': 'vimeo',
             'token': token,
-        })
+        }))
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Cookie', 'vuid=%s' % vuid)

From bfdf891fd36811909aa5d83dc0614eacbb634fcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:09:24 +0600
Subject: [PATCH 1752/2145] [vimeo] Fix non-ASCII album passwords

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 46fb36f21..b608740b8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -489,7 +489,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         token, vuid = self._extract_xsrft_and_vuid(webpage)
         fields['token'] = token
         fields['password'] = password
-        post = urlencode_postdata(fields)
+        post = urlencode_postdata(encode_dict(fields))
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)

From fd8102820c4d14fdb1ff7e090553211717012f67 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:09:55 +0800
Subject: [PATCH 1753/2145] [democracynow] Rename js to json_data

---
 youtube_dl/extractor/democracynow.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 70c364e8b..72fc75d80 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -40,7 +40,7 @@ class DemocracynowIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
         description = self._og_search_description(webpage)
 
-        js = self._parse_json(self._search_regex(
+        json_data = self._parse_json(self._search_regex(
             r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
             display_id)
         video_id = None
@@ -56,19 +56,19 @@ class DemocracynowIE(InfoExtractor):
             subtitles[lang].append(info_dict)
 
         # chapter_file are not subtitles
-        if 'caption_file' in js:
+        if 'caption_file' in json_data:
             add_subtitle_item(default_lang, {
-                'url': compat_urlparse.urljoin(url, js['caption_file']),
+                'url': compat_urlparse.urljoin(url, json_data['caption_file']),
             })
 
-        for subtitle_item in js.get('captions', []):
+        for subtitle_item in json_data.get('captions', []):
             lang = subtitle_item.get('language', '').lower() or default_lang
             add_subtitle_item(lang, {
                 'url': compat_urlparse.urljoin(url, subtitle_item['url']),
             })
 
         for key in ('file', 'audio', 'video'):
-            media_url = js.get(key, '')
+            media_url = json_data.get(key, '')
             if not media_url:
                 continue
             media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
@@ -81,7 +81,7 @@ class DemocracynowIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': js.get('title'),
+            'title': json_data.get('title'),
             'description': description,
             'subtitles': subtitles,
             'formats': formats,

From 0aeb9a106e1aad37967e0ee666ed816a7d5eb7c2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:13:00 +0800
Subject: [PATCH 1754/2145] [democracynow] Prevent required fields to be None

---
 youtube_dl/extractor/democracynow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 72fc75d80..6cd395e11 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -80,8 +80,8 @@ class DemocracynowIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
-            'title': json_data.get('title'),
+            'id': video_id or display_id,
+            'title': json_data['title'],
             'description': description,
             'subtitles': subtitles,
             'formats': formats,

From 66d041f250f7d3e0c4d501e3b98721f2c6588c35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:53:30 +0800
Subject: [PATCH 1755/2145] [test/subtitles] Add test for DemocracynowIE

---
 test/test_subtitles.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 0343967d9..75f0ea75f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -28,6 +28,7 @@ from youtube_dl.extractor import (
     ThePlatformFeedIE,
     RTVEALaCartaIE,
     FunnyOrDieIE,
+    DemocracynowIE,
 )
 
 
@@ -346,5 +347,25 @@ class TestFunnyOrDieSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4')
 
 
+class TestDemocracynowSubtitles(BaseTestSubtitles):
+    url = 'http://www.democracynow.org/shows/2015/7/3'
+    IE = DemocracynowIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+
+    def test_subtitles_in_page(self):
+        self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+
+
 if __name__ == '__main__':
     unittest.main()

From ad607563a2fbb5275ea39f7a052c09ffa232e271 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:46:26 +0600
Subject: [PATCH 1756/2145] [globo] Separate article extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/globo.py    | 140 +++++++++++++++++--------------
 2 files changed, 79 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 10286aa88..94150a28f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -212,7 +212,10 @@ from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
-from .globo import GloboIE
+from .globo import (
+    GloboIE,
+    GloboArticleIE,
+)
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 33d6432a6..828e40d76 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -18,75 +18,52 @@ from ..utils import (
 
 
 class GloboIE(InfoExtractor):
-    _VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
+    _VALID_URL = '(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P<id>\d{7,})'
 
     _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
     _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'
 
-    _VIDEOID_REGEXES = [
-        r'\bdata-video-id="(\d+)"',
-        r'\bdata-player-videosids="(\d+)"',
-        r'<div[^>]+\bid="(\d+)"',
-    ]
-
     _RESIGN_EXPIRATION = 86400
 
-    _TESTS = [
-        {
-            'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
-            'md5': '03ebf41cb7ade43581608b7d9b71fab0',
-            'info_dict': {
-                'id': '3654973',
-                'ext': 'mp4',
-                'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
-                'duration': 251.585,
-                'uploader': 'SporTV',
-                'uploader_id': 698,
-                'like_count': int,
-            }
-        },
-        {
-            'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
-            'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
-            'info_dict': {
-                'id': '3607726',
-                'ext': 'mp4',
-                'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
-                'duration': 103.204,
-                'uploader': 'Globo.com',
-                'uploader_id': 265,
-                'like_count': int,
-            }
-        },
-        {
-            'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
-            'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
-            'info_dict': {
-                'id': '3652183',
-                'ext': 'mp4',
-                'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
-                'duration': 110.711,
-                'uploader': 'Rede Globo',
-                'uploader_id': 196,
-                'like_count': int,
-            }
-        },
-        {
-            'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
-            'md5': 'c1defca721ce25b2354e927d3e4b3dec',
-            'info_dict': {
-                'id': '3928201',
-                'ext': 'mp4',
-                'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
-                'duration': 1472.906,
-                'uploader': 'Canal Brasil',
-                'uploader_id': 705,
-                'like_count': int,
-            }
-        },
-    ]
+    _TESTS = [{
+        'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
+        'md5': '03ebf41cb7ade43581608b7d9b71fab0',
+        'info_dict': {
+            'id': '3654973',
+            'ext': 'mp4',
+            'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
+            'duration': 251.585,
+            'uploader': 'SporTV',
+            'uploader_id': 698,
+            'like_count': int,
+        }
+    }, {
+        'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
+        'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
+        'info_dict': {
+            'id': '3607726',
+            'ext': 'mp4',
+            'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
+            'duration': 103.204,
+            'uploader': 'Globo.com',
+            'uploader_id': 265,
+            'like_count': int,
+        }
+    }, {
+        'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
+        'md5': 'c1defca721ce25b2354e927d3e4b3dec',
+        'info_dict': {
+            'id': '3928201',
+            'ext': 'mp4',
+            'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
+            'duration': 1472.906,
+            'uploader': 'Canal Brasil',
+            'uploader_id': 705,
+            'like_count': int,
+        }
+    }]
 
-    class MD5():
+    class MD5:
         HEX_FORMAT_LOWERCASE = 0
         HEX_FORMAT_UPPERCASE = 1
         BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''
@@ -353,9 +330,6 @@ class GloboIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
-
         video = self._download_json(
             self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
 
@@ -417,3 +391,39 @@ class GloboIE(InfoExtractor):
             'like_count': like_count,
             'formats': formats
         }
+
+
+class GloboArticleIE(InfoExtractor):
+    _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+
+    _VIDEOID_REGEXES = [
+        r'\bdata-video-id=["\'](\d{7,})',
+        r'\bdata-player-videosids=["\'](\d{7,})',
+        r'\bvideosIDs\s*:\s*["\'](\d{7,})',
+        r'\bdata-id=["\'](\d{7,})',
+        r'<div[^>]+\bid=["\'](\d{7,})',
+    ]
+
+    _TEST = {
+        'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
+        'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
+        'info_dict': {
+            'id': '3652183',
+            'ext': 'mp4',
+            'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
+            'duration': 110.711,
+            'uploader': 'Rede Globo',
+            'uploader_id': 196,
+            'like_count': int,
+        }
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if GloboIE.suitable(url) else super(GloboArticleIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
+        return self.url_result('globo:%s' % video_id, 'Globo')

From e3778cce0e912f803ea10cb806406f7fcafe840f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:51:19 +0600
Subject: [PATCH 1757/2145] [globo] Improve m3u8 extraction

---
 youtube_dl/extractor/globo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 828e40d76..c28899011 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -367,7 +367,10 @@ class GloboIE(InfoExtractor):
             resource_url = resource['url']
             signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
             if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(signed_url, resource_id, 'mp4'))
+                m3u8_formats = self._extract_m3u8_formats(
+                    signed_url, resource_id, 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'url': signed_url,

From c3459d24f16056e8ae8f982db2a10871ef18e80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:53:21 +0600
Subject: [PATCH 1758/2145] [globo] Skip unsupported smooth streaming

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index c28899011..ec451bb07 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -338,7 +338,7 @@ class GloboIE(InfoExtractor):
         formats = []
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            if not resource_id:
+            if not resource_id or resource_id.endswith('manifest'):
                 continue
 
             security = self._download_json(

From 5d235ca7f66af1f82c1a4d753d238f48fc3afa40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:55:39 +0600
Subject: [PATCH 1759/2145] [globo] Prefer native m3u8

---
 youtube_dl/extractor/globo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index ec451bb07..2a805cbb2 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -368,7 +368,8 @@ class GloboIE(InfoExtractor):
             signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
             if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
                 m3u8_formats = self._extract_m3u8_formats(
-                    signed_url, resource_id, 'mp4', m3u8_id='hls', fatal=False)
+                    signed_url, resource_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             else:

From b4ef6a0038657c1adde565df947e42ad1e1b4195 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:01:27 +0600
Subject: [PATCH 1760/2145] [globo] Remove non available test

---
 youtube_dl/extractor/globo.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 2a805cbb2..8aada01dc 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -26,18 +26,6 @@ class GloboIE(InfoExtractor):
     _RESIGN_EXPIRATION = 86400
 
     _TESTS = [{
-        'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
-        'md5': '03ebf41cb7ade43581608b7d9b71fab0',
-        'info_dict': {
-            'id': '3654973',
-            'ext': 'mp4',
-            'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
-            'duration': 251.585,
-            'uploader': 'SporTV',
-            'uploader_id': 698,
-            'like_count': int,
-        }
-    }, {
         'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
         'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
         'info_dict': {

From aebb42d32b608eaffb424e5e7c22f1b68a491e3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:01:55 +0600
Subject: [PATCH 1761/2145] [globo] Remove like count

It's no longer provided
---
 youtube_dl/extractor/globo.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 8aada01dc..dc89e46ac 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -35,7 +35,6 @@ class GloboIE(InfoExtractor):
             'duration': 103.204,
             'uploader': 'Globo.com',
             'uploader_id': 265,
-            'like_count': int,
         }
     }, {
         'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
@@ -47,7 +46,6 @@ class GloboIE(InfoExtractor):
             'duration': 1472.906,
             'uploader': 'Canal Brasil',
             'uploader_id': 705,
-            'like_count': int,
         }
     }]
 
@@ -370,7 +368,6 @@ class GloboIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = float_or_none(video.get('duration'), 1000)
-        like_count = int_or_none(video.get('likes'))
         uploader = video.get('channel')
         uploader_id = video.get('channel_id')
 
@@ -380,7 +377,6 @@ class GloboIE(InfoExtractor):
             'duration': duration,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'like_count': like_count,
             'formats': formats
         }
 
@@ -406,7 +402,6 @@ class GloboArticleIE(InfoExtractor):
             'duration': 110.711,
             'uploader': 'Rede Globo',
             'uploader_id': 196,
-            'like_count': int,
         }
     }
 

From a4a6b7b80f18680ee0a8bba50a24c58edd3f2a73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:03:45 +0600
Subject: [PATCH 1762/2145] [globo] Improve http formats

---
 youtube_dl/extractor/globo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index dc89e46ac..64622aa5c 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -361,8 +361,8 @@ class GloboIE(InfoExtractor):
             else:
                 formats.append({
                     'url': signed_url,
-                    'format_id': resource_id,
-                    'height': resource.get('height'),
+                    'format_id': 'http-%s' % resource_id,
+                    'height': int_or_none(resource.get('height')),
                 })
 
         self._sort_formats(formats)

From 264cd00fff4f6d7063d43e1d476de46901bd9c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:10:45 +0600
Subject: [PATCH 1763/2145] [globo] Update tests

---
 youtube_dl/extractor/globo.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 64622aa5c..0337256ed 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -35,18 +35,30 @@ class GloboIE(InfoExtractor):
             'duration': 103.204,
             'uploader': 'Globo.com',
             'uploader_id': 265,
-        }
+        },
+    }, {
+        'url': 'http://globoplay.globo.com/v/4581987/',
+        'md5': 'f36a1ecd6a50da1577eee6dd17f67eff',
+        'info_dict': {
+            'id': '4581987',
+            'ext': 'mp4',
+            'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
+            'duration': 137.973,
+            'uploader': 'Rede Globo',
+            'uploader_id': 196,
+        },
+    }, {
+        'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://globosatplay.globo.com/globonews/v/4472924/',
+        'only_matching': True,
+    }, {
+        'url': 'http://globotv.globo.com/t/programa/v/clipe-sexo-e-as-negas-adeus/3836166/',
+        'only_matching': True,
     }, {
         'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
-        'md5': 'c1defca721ce25b2354e927d3e4b3dec',
-        'info_dict': {
-            'id': '3928201',
-            'ext': 'mp4',
-            'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
-            'duration': 1472.906,
-            'uploader': 'Canal Brasil',
-            'uploader_id': 705,
-        }
+        'only_matching': True,
     }]
 
     class MD5:

From e7d34c03f200e178e9d6dfe4ae3f6856e382a4b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:12:42 +0600
Subject: [PATCH 1764/2145] [globo] Force uploader id to be string

---
 youtube_dl/extractor/globo.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 0337256ed..6c0fc54de 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -14,6 +14,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    str_or_none,
 )
 
 
@@ -34,7 +35,7 @@ class GloboIE(InfoExtractor):
             'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
             'duration': 103.204,
             'uploader': 'Globo.com',
-            'uploader_id': 265,
+            'uploader_id': '265',
         },
     }, {
         'url': 'http://globoplay.globo.com/v/4581987/',
@@ -45,7 +46,7 @@ class GloboIE(InfoExtractor):
             'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
             'duration': 137.973,
             'uploader': 'Rede Globo',
-            'uploader_id': 196,
+            'uploader_id': '196',
         },
     }, {
         'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
@@ -381,7 +382,7 @@ class GloboIE(InfoExtractor):
 
         duration = float_or_none(video.get('duration'), 1000)
         uploader = video.get('channel')
-        uploader_id = video.get('channel_id')
+        uploader_id = str_or_none(video.get('channel_id'))
 
         return {
             'id': video_id,

From c13722480bebfb1fc33169516790df2e99b3e499 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:13:35 +0600
Subject: [PATCH 1765/2145] [globo:article] Fix test

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 6c0fc54de..5883be704 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -414,7 +414,7 @@ class GloboArticleIE(InfoExtractor):
             'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
             'duration': 110.711,
             'uploader': 'Rede Globo',
-            'uploader_id': 196,
+            'uploader_id': '196',
         }
     }
 

From 5d501a0901c36695c9d6ca3958ac4ccfdea90954 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:42:11 +0600
Subject: [PATCH 1766/2145] [globo] Add more tests

---
 youtube_dl/extractor/globo.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 5883be704..c65ef6bcf 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -60,6 +60,9 @@ class GloboIE(InfoExtractor):
     }, {
         'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
         'only_matching': True,
+    }, {
+        'url': 'http://canaloff.globo.com/programas/desejar-profundo/videos/4518560.html',
+        'only_matching': True,
     }]
 
     class MD5:
@@ -405,7 +408,7 @@ class GloboArticleIE(InfoExtractor):
         r'<div[^>]+\bid=["\'](\d{7,})',
     ]
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
         'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
         'info_dict': {
@@ -416,7 +419,13 @@ class GloboArticleIE(InfoExtractor):
             'uploader': 'Rede Globo',
             'uploader_id': '196',
         }
-    }
+    }, {
+        'url': 'http://gq.globo.com/Prazeres/Poder/noticia/2015/10/all-o-desafio-assista-ao-segundo-capitulo-da-serie.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://gshow.globo.com/programas/tv-xuxa/O-Programa/noticia/2014/01/xuxa-e-junno-namoram-muuuito-em-luau-de-zeze-di-camargo-e-luciano.html',
+        'only_matching': True,
+    }]
 
     @classmethod
     def suitable(cls, url):

From 17d1900581ffd12866e56640080ce340d99149a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:57:46 +0600
Subject: [PATCH 1767/2145] [vk] Fix view count extraction (Closes #7353)

---
 youtube_dl/extractor/vk.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 765e9e6fd..01960b827 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -281,9 +281,13 @@ class VKIE(InfoExtractor):
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
-        view_count = str_to_int(self._search_regex(
-            r'"mv_views_count_number"[^>]*>([\d,.]+) views<',
-            info_page, 'view count', fatal=False))
+        view_count = None
+        views = self._html_search_regex(
+            r'"mv_views_count_number"[^>]*>(.+?\bviews?)<',
+            info_page, 'view count', fatal=False)
+        if views:
+            view_count = str_to_int(self._search_regex(
+                r'([\d,.]+)', views, 'view count', fatal=False))
 
         formats = [{
             'format_id': k,

From cb5a470635ea2ad91f18d33e391979aabb0755fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 4 Nov 2015 16:18:51 +0100
Subject: [PATCH 1768/2145] [vimeo] Remove unused import

---
 youtube_dl/extractor/vimeo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b608740b8..ca716c8f5 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,6 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
 )

From 44b2264feae331eeb34e83eed1387def3d61a437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 22:12:24 +0600
Subject: [PATCH 1769/2145] [youtube] Prefer video_info with token available

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d7eda7aa7..5eeb3c663 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1107,6 +1107,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:
+                        if 'token' not in video_info:
+                            video_info = get_video_info
                         break
         if 'token' not in video_info:
             if 'reason' in video_info:

From 89ea063eebae84792a7ccb968533ff8bf6a41d56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 22:49:23 +0600
Subject: [PATCH 1770/2145] [youtube] Clarify rationale for preferring a video
 info with token (#7362)

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5eeb3c663..e2a43299f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1107,6 +1107,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:
+                        # Different get_video_info requests may report different results, e.g.
+                        # some may report video unavailability, but some may serve it without
+                        # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
+                        # the original webpage as well as el=info and el=embedded get_video_info
+                        # requests report video unavailability due to geo restriction while
+                        # el=detailpage succeeds and returns valid data). This is probably
+                        # due to YouTube measures against IP ranges of hosting providers.
+                        # Working around by preferring the first succeeded video_info containing
+                        # the token if no such video_info yet was found.
                         if 'token' not in video_info:
                             video_info = get_video_info
                         break

From f93ded98522cc1272a8d2210738937132292afc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 01:54:49 +0600
Subject: [PATCH 1771/2145] [prosiebensat1] Add support for .ch domains (Closes
 #7365)

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index effcf1db3..baa54a3af 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -20,7 +20,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at)|ran\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {

From b15c44cd36831f175e9dd4081b82beb8075790b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:51:30 +0600
Subject: [PATCH 1772/2145] [periscope] Add support for videos with
 broadcast_id (Closes #7359)

---
 youtube_dl/extractor/periscope.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 8ad936758..0f9d7576f 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -27,9 +27,10 @@ class PeriscopeIE(InfoExtractor):
         'skip': 'Expires in 24 hours',
     }
 
-    def _call_api(self, method, token):
+    def _call_api(self, method, value):
+        attribute = 'token' if len(value) > 13 else 'broadcast_id'
         return self._download_json(
-            'https://api.periscope.tv/api/v2/%s?token=%s' % (method, token), token)
+            'https://api.periscope.tv/api/v2/%s?%s=%s' % (method, attribute, value), value)
 
     def _real_extract(self, url):
         token = self._match_id(url)

From 2549e113b8750a493917436d4fd15ed74a1a4983 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:55:53 +0600
Subject: [PATCH 1773/2145] [periscope] Add test for broadcast_id based URL

---
 youtube_dl/extractor/periscope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 0f9d7576f..7621d9e99 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,7 +12,7 @@ from ..utils import parse_iso8601
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
         'md5': '65b57957972e503fcbbaeed8f4fa04ca',
         'info_dict': {
@@ -25,7 +25,10 @@ class PeriscopeIE(InfoExtractor):
             'uploader_id': '1465763',
         },
         'skip': 'Expires in 24 hours',
-    }
+    }, {
+        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
+        'only_matching': True,
+    }]
 
     def _call_api(self, method, value):
         attribute = 'token' if len(value) > 13 else 'broadcast_id'

From 53472df85793cc89deb779c2ffc3ae1f47292fd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:56:44 +0600
Subject: [PATCH 1774/2145] [periscope] Add note on where to find alive example
 URLs

---
 youtube_dl/extractor/periscope.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 7621d9e99..887c8020d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,6 +12,7 @@ from ..utils import parse_iso8601
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
         'md5': '65b57957972e503fcbbaeed8f4fa04ca',

From b3613d36da14ab527166326707c0f911d192144d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 04:37:51 +0600
Subject: [PATCH 1775/2145] [YoutubeDL] Sanitize path after output template
 substitution (Closes #7367)

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 12977bf80..1783ce01b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -572,7 +572,7 @@ class YoutubeDL(object):
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
+            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787
@@ -580,7 +580,7 @@ class YoutubeDL(object):
             # to workaround encoding issues with subprocess on python2 @ Windows
             if sys.version_info < (3, 0) and sys.platform == 'win32':
                 filename = encodeFilename(filename, True).decode(preferredencoding())
-            return filename
+            return sanitize_path(filename)
         except ValueError as err:
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None

From 6953d8e95a78e83f087693b7353baab96b09fbdd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 02:09:55 +0100
Subject: [PATCH 1776/2145] [miomio] fix info extraction (fixes #7366)

---
 youtube_dl/extractor/miomio.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index a784fc5fb..3f812e005 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import (
     xpath_text,
     int_or_none,
@@ -60,10 +61,12 @@ class MioMioIE(InfoExtractor):
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
             video_id)
 
-        # the following xml contains the actual configuration information on the video file(s)
-        vid_config = self._download_xml(
+        vid_config_request = compat_urllib_request.Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
-            video_id)
+            headers={'Referer': 'http://www.miomio.tv/mioplayer/mioplayer-v3.0.swf'})
+
+        # the following xml contains the actual configuration information on the video file(s)
+        vid_config = self._download_xml(vid_config_request, video_id)
 
         http_headers = {
             'Referer': 'http://www.miomio.tv%s' % mioplayer_path,

From e68dd1921ad7528d225a8571066f99b9934b6a06 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 06:33:05 +0100
Subject: [PATCH 1777/2145] [miomio] use the formats urls headers for
 downloading xml

---
 youtube_dl/extractor/miomio.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 3f812e005..6f40bf1b9 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -52,6 +52,8 @@ class MioMioIE(InfoExtractor):
         mioplayer_path = self._search_regex(
             r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
 
+        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path,}
+
         xml_config = self._search_regex(
             r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
             webpage, 'xml config')
@@ -63,15 +65,11 @@ class MioMioIE(InfoExtractor):
 
         vid_config_request = compat_urllib_request.Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
-            headers={'Referer': 'http://www.miomio.tv/mioplayer/mioplayer-v3.0.swf'})
+            headers=http_headers)
 
         # the following xml contains the actual configuration information on the video file(s)
         vid_config = self._download_xml(vid_config_request, video_id)
 
-        http_headers = {
-            'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
-        }
-
         if not int_or_none(xpath_text(vid_config, 'timelength')):
             raise ExtractorError('Unable to load videos!', expected=True)
 

From 5003e4283b35acb82ea9793d91bc3cd1ee679f86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:06:44 +0600
Subject: [PATCH 1778/2145] [ndr] Relax _VALID_URL (Closes #7383)

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index ba06d8a98..a2b51ccb3 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -23,7 +23,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',

From 01003d072c20c2ed095930d87c5ce3d5610e66b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:07:52 +0600
Subject: [PATCH 1779/2145] [ndr] Add test for #7383

---
 youtube_dl/extractor/ndr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index a2b51ccb3..0be866681 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -78,6 +78,9 @@ class NDRIE(NDRBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.ndr.de/Fettes-Brot-Ferris-MC-und-Thees-Uhlmann-live-on-stage,festivalsommer116.html',
+        'only_matching': True,
     }]
 
     def _extract_embed(self, webpage, display_id):

From 1e2eb4b40d46f39d15c067657ecad16fa3b2121d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:08:21 +0600
Subject: [PATCH 1780/2145] [njoy] Relax _VALID_URL

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 0be866681..7043c7e0f 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -105,7 +105,7 @@ class NDRIE(NDRBaseIE):
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',

From 81413c01651eddcc5180af379f2ce3689a376051 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:08:52 +0600
Subject: [PATCH 1781/2145] [ndr:embed] Relax _VALID_URL

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 7043c7e0f..477ce4e6b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -238,7 +238,7 @@ class NDREmbedBaseIE(InfoExtractor):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',

From 92366d189ef280b8ba0057930c54aa14b0ecdd24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:09:17 +0600
Subject: [PATCH 1782/2145] [njoy:embed] Relax _VALID_URL

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 477ce4e6b..16213eed9 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -332,7 +332,7 @@ class NDREmbedIE(NDREmbedBaseIE):
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'njoy:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
     _TESTS = [{
         # httpVideo
         'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',

From deb85c32bbd32e8d280e1919432a11c0bdaa26bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:56:31 +0600
Subject: [PATCH 1783/2145] [postprocessor/ffmpeg] Use ffmpeg as prefix since
 it's used all over the places (Closes #7371)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4f320e124..5ed723bc6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -272,7 +272,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             return [], information
 
         try:
-            self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
+            self._downloader.to_screen('[ffmpeg] Destination: ' + new_path)
             self.run_ffmpeg(path, new_path, acodec, more_opts)
         except AudioConversionError as e:
             raise PostProcessingError(

From 179ffab69c3359ab7d0a7b0a2b63c94d8c70af67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:06:13 +0600
Subject: [PATCH 1784/2145] [lynda:course] Force log out (Closes #7361)

---
 youtube_dl/extractor/lynda.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5c973e75c..67f2025de 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -82,6 +82,11 @@ class LyndaBaseIE(InfoExtractor):
                         expected=True)
             raise ExtractorError('Unable to log in')
 
+    def _logout(self):
+        self._download_webpage(
+            'http://www.lynda.com/ajax/logout.aspx', None,
+            'Logging out', 'Unable to log out', fatal=False)
+
 
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
@@ -210,6 +215,8 @@ class LyndaCourseIE(LyndaBaseIE):
             course_id, 'Downloading course JSON')
         course_json = json.loads(page)
 
+        self._logout()
+
         if 'Status' in course_json and course_json['Status'] == 'NotFound':
             raise ExtractorError(
                 'Course %s does not exist' % course_id, expected=True)

From 71bb016160744a80fecaadf5b75b0dc2b1e8089b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:10:07 +0600
Subject: [PATCH 1785/2145] [lynda:course] Modernize and make more robust

---
 youtube_dl/extractor/lynda.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 67f2025de..98474ded9 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -210,14 +210,13 @@ class LyndaCourseIE(LyndaBaseIE):
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
 
-        page = self._download_webpage(
+        course = self._download_json(
             'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
             course_id, 'Downloading course JSON')
-        course_json = json.loads(page)
 
         self._logout()
 
-        if 'Status' in course_json and course_json['Status'] == 'NotFound':
+        if course.get('Status') == 'NotFound':
             raise ExtractorError(
                 'Course %s does not exist' % course_id, expected=True)
 
@@ -227,12 +226,14 @@ class LyndaCourseIE(LyndaBaseIE):
         # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
         # by single video API anymore
 
-        for chapter in course_json['Chapters']:
-            for video in chapter['Videos']:
-                if video['HasAccess'] is False:
+        for chapter in course['Chapters']:
+            for video in chapter.get('Videos', []):
+                if video.get('HasAccess') is False:
                     unaccessible_videos += 1
                     continue
-                videos.append(video['ID'])
+                video_id = video.get('ID')
+                if video_id:
+                    videos.append(video_id)
 
         if unaccessible_videos > 0:
             self._downloader.report_warning(
@@ -245,6 +246,6 @@ class LyndaCourseIE(LyndaBaseIE):
                 'Lynda')
             for video_id in videos]
 
-        course_title = course_json['Title']
+        course_title = course.get('Title')
 
         return self.playlist_result(entries, course_id, course_title)

From ea8ed40b2fb70fc2f01aba475128821078873d46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:24:39 +0600
Subject: [PATCH 1786/2145] [lynda] Modernize and make more robust

---
 youtube_dl/extractor/lynda.py | 52 ++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 98474ded9..c8a16842e 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -113,51 +113,47 @@ class LyndaIE(LyndaBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        page = self._download_webpage(
+        video = self._download_json(
             'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
             video_id, 'Downloading video JSON')
-        video_json = json.loads(page)
 
-        if 'Status' in video_json:
+        if 'Status' in video:
             raise ExtractorError(
-                'lynda returned error: %s' % video_json['Message'], expected=True)
+                'lynda returned error: %s' % video['Message'], expected=True)
 
-        if video_json['HasAccess'] is False:
+        if video.get('HasAccess') is False:
             self.raise_login_required('Video %s is only available for members' % video_id)
 
-        video_id = compat_str(video_json['ID'])
-        duration = video_json['DurationInSeconds']
-        title = video_json['Title']
+        video_id = compat_str(video.get('ID') or video_id)
+        duration = int_or_none(video.get('DurationInSeconds'))
+        title = video['Title']
 
         formats = []
 
-        fmts = video_json.get('Formats')
+        fmts = video.get('Formats')
         if fmts:
-            formats.extend([
-                {
-                    'url': fmt['Url'],
-                    'ext': fmt['Extension'],
-                    'width': fmt['Width'],
-                    'height': fmt['Height'],
-                    'filesize': fmt['FileSize'],
-                    'format_id': str(fmt['Resolution'])
-                } for fmt in fmts])
+            formats.extend([{
+                'url': f['Url'],
+                'ext': f.get('Extension'),
+                'width': int_or_none(f.get('Width')),
+                'height': int_or_none(f.get('Height')),
+                'filesize': int_or_none(f.get('FileSize')),
+                'format_id': compat_str(f.get('Resolution')) if f.get('Resolution') else None,
+            } for f in fmts if f.get('Url')])
 
-        prioritized_streams = video_json.get('PrioritizedStreams')
+        prioritized_streams = video.get('PrioritizedStreams')
         if prioritized_streams:
             for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
-                formats.extend([
-                    {
-                        'url': video_url,
-                        'width': int_or_none(format_id),
-                        'format_id': '%s-%s' % (prioritized_stream_id, format_id),
-                    } for format_id, video_url in prioritized_stream.items()
-                ])
+                formats.extend([{
+                    'url': video_url,
+                    'width': int_or_none(format_id),
+                    'format_id': '%s-%s' % (prioritized_stream_id, format_id),
+                } for format_id, video_url in prioritized_stream.items()])
 
         self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
-        subtitles = self.extract_subtitles(video_id, page)
+        subtitles = self.extract_subtitles(video_id)
 
         return {
             'id': video_id,
@@ -188,7 +184,7 @@ class LyndaIE(LyndaBaseIE):
         if srt:
             return srt
 
-    def _get_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id):
         url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
         subs = self._download_json(url, None, False)
         if subs:

From ae4ddf9efae816f4d52fc584c93e4f0e3c79c410 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:27:38 +0600
Subject: [PATCH 1787/2145] [lynda] PEP 8

---
 youtube_dl/extractor/lynda.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index c8a16842e..9a207b2cd 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -227,9 +227,8 @@ class LyndaCourseIE(LyndaBaseIE):
                 if video.get('HasAccess') is False:
                     unaccessible_videos += 1
                     continue
-                video_id = video.get('ID')
-                if video_id:
-                    videos.append(video_id)
+                if video.get('ID'):
+                    videos.append(video['ID'])
 
         if unaccessible_videos > 0:
             self._downloader.report_warning(

From 472404953a22811cc8156da110ea872a924f1f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:28:14 +0600
Subject: [PATCH 1788/2145] [miomio] PEP 8

---
 youtube_dl/extractor/miomio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 6f40bf1b9..ce391c759 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -52,7 +52,7 @@ class MioMioIE(InfoExtractor):
         mioplayer_path = self._search_regex(
             r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
 
-        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path,}
+        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
 
         xml_config = self._search_regex(
             r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',

From 0fa6b17dccd2347cb0611651fc04e36839d33a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:45:26 +0600
Subject: [PATCH 1789/2145] [pbs] Simplify and speed up player URL search

---
 youtube_dl/extractor/pbs.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3448736a2..7b868d057 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -191,9 +191,13 @@ class PBSIE(InfoExtractor):
             if media_id:
                 return media_id, presumptive_id, upload_date
 
-            url = self._search_regex(
-                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].*?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
-                webpage, 'player URL')
+            for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
+                url = self._search_regex(
+                    r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,
+                    'player URL', default=None, group='url')
+                if url:
+                    break
+
             mobj = re.match(self._VALID_URL, url)
 
         player_id = mobj.group('player_id')

From 686f98816ecbbcb224d1336682688b05cdb051a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 00:39:16 +0600
Subject: [PATCH 1790/2145] [pbs] Add support for flp frontlines (Closes #7369)

---
 youtube_dl/extractor/pbs.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 7b868d057..3169e9c3f 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    strip_jsonp,
     unified_strdate,
     US_RATINGS,
 )
@@ -191,6 +192,23 @@ class PBSIE(InfoExtractor):
             if media_id:
                 return media_id, presumptive_id, upload_date
 
+            # Fronline video embedded via flp
+            video_id = self._search_regex(
+                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')
+            if video_id:
+                # pkg_id calculation is reverse engineered from
+                # http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js
+                prg_id = self._search_regex(
+                    r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')[7:]
+                if 'q' in prg_id:
+                    prg_id = prg_id.split('q')[1]
+                prg_id = int(prg_id, 16)
+                getdir = self._download_json(
+                    'http://www.pbs.org/wgbh/pages/frontline/.json/getdir/getdir%d.json' % prg_id,
+                    presumptive_id, 'Downloading getdir JSON',
+                    transform_source=strip_jsonp)
+                return getdir['mid'], presumptive_id, upload_date
+
             for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
                 url = self._search_regex(
                     r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,

From 8b6d9406db1d3361b006016e6aace54b05cb6fea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 00:42:30 +0600
Subject: [PATCH 1791/2145] [pbs] Add test for flp frontline embeds

---
 youtube_dl/extractor/pbs.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3169e9c3f..a690f9c29 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -154,6 +154,22 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            # Frontline video embedded via flp2012.js
+            'url': 'http://www.pbs.org/wgbh/pages/frontline/the-atomic-artists',
+            'info_dict': {
+                'id': '2070868960',
+                'display_id': 'the-atomic-artists',
+                'ext': 'mp4',
+                'title': 'FRONTLINE - The Atomic Artists',
+                'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+                'duration': 723,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
     _ERRORS = {

From 21d0c33ecde573db961b97f5f0c37ba9d3c02ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 01:08:40 +0600
Subject: [PATCH 1792/2145] [pbs] Make flp embed lookup non fatal

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index a690f9c29..8fb9b1849 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -210,7 +210,7 @@ class PBSIE(InfoExtractor):
 
             # Fronline video embedded via flp
             video_id = self._search_regex(
-                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')
+                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid', default=None)
             if video_id:
                 # pkg_id calculation is reverse engineered from
                 # http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js

From ee223abb88263bdda2d92c4b2139d1dca60ba3ae Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Tue, 3 Nov 2015 19:13:27 -0600
Subject: [PATCH 1793/2145] [vidzi] fixed. finds url from hash and host in
 script

Closes #7386.
---
 youtube_dl/extractor/vidzi.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 08a5a7b8d..2ba9f31df 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -20,8 +20,14 @@ class VidziIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        video_url = self._html_search_regex(
-            r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
+        video_host = self._html_search_regex(
+            r'id=\'vplayer\'><img src="http://(.*?)/i', webpage,
+            'video host')
+        video_hash = self._html_search_regex(
+            r'\|([a-z0-9]+)\|hls\|type', webpage, 'video_hash')
+        ext = self._html_search_regex(
+            r'\|tracks\|([a-z0-9]+)\|', webpage, 'video ext')
+        video_url = 'http://' + video_host + '/' + video_hash + '/v.' + ext
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 

From 5d0f84d32cc038dd71673987cb6efaa85e953474 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 06:23:00 +0600
Subject: [PATCH 1794/2145] [beeg] Skip empty URLs (Closes #7392)

---
 youtube_dl/extractor/beeg.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index e6c928699..61bc2f744 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -33,6 +33,8 @@ class BeegIE(InfoExtractor):
 
         formats = []
         for format_id, video_url in video.items():
+            if not video_url:
+                continue
             height = self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None)
             if not height:

From 5214f1e31d5e5ba692fb1ed4803ff71ef4e480e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 19:25:59 +0600
Subject: [PATCH 1795/2145] [crunchyroll] Fix title extraction (Closes #7396)

---
 youtube_dl/extractor/crunchyroll.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 0c9b8ca02..4243f3e2e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -287,7 +287,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
             self.raise_login_required()
 
-        video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
+        video_title = self._html_search_regex(
+            r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
+            webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
         if not video_description:

From 2c740cf28d257d2a915195e7cc60f83e6690d2cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 19:29:09 +0600
Subject: [PATCH 1796/2145] [crunchyroll] Simplify description extraction

---
 youtube_dl/extractor/crunchyroll.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4243f3e2e..9aa5d58b4 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -291,9 +291,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
             webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
-        video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
-        if not video_description:
-            video_description = None
+        video_description = self._html_search_regex(
+            r'"description":"([^"]+)', webpage, 'video_description', default=None)
         video_upload_date = self._html_search_regex(
             [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
             webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)

From 6d02b9a392d39c114d3fb58bf7965f62196ccecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 20:02:39 +0600
Subject: [PATCH 1797/2145] [crunchyroll] Fix description extraction

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 9aa5d58b4..6e5999c72 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -21,6 +21,7 @@ from ..utils import (
     bytes_to_intlist,
     intlist_to_bytes,
     int_or_none,
+    lowercase_escape,
     remove_end,
     unified_strdate,
     urlencode_postdata,
@@ -104,7 +105,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'id': '589804',
             'ext': 'flv',
             'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
-            'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
+            'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'Danny Choo Network',
             'upload_date': '20120213',
@@ -292,7 +293,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(
-            r'"description":"([^"]+)', webpage, 'video_description', default=None)
+            r'<script[^>]*>\s*.+?\[media_id=%s\].+?"description"\s*:\s*"([^"]+)' % video_id,
+            webpage, 'description', default=None)
+        if video_description:
+            video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
         video_upload_date = self._html_search_regex(
             [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
             webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)

From cff551c0b0ed8eb55c1ab63ec669c07a51aa4998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 7 Nov 2015 18:43:22 +0100
Subject: [PATCH 1798/2145] [googleplus] Fix extraction of formats

---
 youtube_dl/extractor/googleplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index fcefe54cd..731bacd67 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -61,7 +61,7 @@ class GooglePlusIE(InfoExtractor):
             'width': int(width),
             'height': int(height),
         } for width, height, video_url in re.findall(
-            r'\d+,(\d+),(\d+),"(https?://redirector\.googlevideo\.com.*?)"', webpage)]
+            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent.com.*?)"', webpage)]
         self._sort_formats(formats)
 
         return {

From ee4337d100f68bbb2ae795101d4c391b522ec753 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 20:16:14 +0100
Subject: [PATCH 1799/2145] [videolecture] add support for multi part videos

---
 youtube_dl/extractor/videolecturesnet.py | 95 +++++++++++++++++-------
 1 file changed, 70 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 649ac9433..351706362 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -10,17 +10,19 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     parse_duration,
+    js_to_json,
+    parse_iso8601,
 )
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'
     IE_NAME = 'videolectures.net'
 
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
-            'id': 'promogram_igor_mekjavic_eng',
+            'id': '20171_part1',
             'ext': 'mp4',
             'title': 'Automatics, robotics and biocybernetics',
             'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
@@ -32,7 +34,7 @@ class VideoLecturesNetIE(InfoExtractor):
         # video with invalid direct format links (HTTP 403)
         'url': 'http://videolectures.net/russir2010_filippova_nlp/',
         'info_dict': {
-            'id': 'russir2010_filippova_nlp',
+            'id': '14891_part1',
             'ext': 'flv',
             'title': 'NLP at Google',
             'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
@@ -46,37 +48,80 @@ class VideoLecturesNetIE(InfoExtractor):
     }, {
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {
-            'id': 'deeplearning2015_montreal',
+            'id': '23181',
             'title': 'Deep Learning Summer School, Montreal 2015',
-            'description': 'md5:90121a40cc6926df1bf04dcd8563ed3b',
+            'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+            'timestamp': 1438560000,
         },
         'playlist_count': 30,
+    }, {
+        # multi part lecture
+        'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
+        'info_dict': {
+            'id': '9737',
+            'title': 'Introduction To Bayesian Inference',
+            'timestamp': 1251622800,
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '9737_part1',
+                'ext': 'wmv',
+                'title': 'Introduction To Bayesian Inference',
+            },
+        }, {
+            'info_dict': {
+                'id': '9737_part2',
+                'ext': 'wmv',
+                'title': 'Introduction To Bayesian Inference',
+            },
+        }],
+        'playlist_count': 2,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        lecture_slug, part = re.match(self._VALID_URL, url).groups()
 
-        smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
+        webpage = self._download_webpage(url, lecture_slug)
 
-        try:
-            smil = self._download_smil(smil_url, video_id)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
-                # Probably a playlist
-                webpage = self._download_webpage(url, video_id)
-                entries = [
-                    self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                    for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
-                playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
-                playlist_description = self._html_search_meta('description', webpage, 'description')
-                return self.playlist_result(entries, video_id, playlist_title, playlist_description)
+        cfg = self._parse_json(self._search_regex(r'cfg\s*:\s*({[^}]+})', webpage, 'cfg'), lecture_slug, js_to_json)
 
-        info = self._parse_smil(smil, smil_url, video_id)
+        lecture_id = str(cfg['obj_id'])
 
-        info['id'] = video_id
+        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (self._proto_relative_url(cfg['livepipe'], 'http:'), lecture_id), lecture_id)['lecture'][0]
 
-        switch = smil.find('.//switch')
-        if switch is not None:
-            info['duration'] = parse_duration(switch.attrib.get('dur'))
+        lecture_info = {
+            'id': lecture_id,
+            'display_id': lecture_slug,
+            'title': lecture_data['title'],
+            'timestamp': parse_iso8601(lecture_data.get('time')),
+            'description': lecture_data.get('description_wiki'),
+            'thumbnail': lecture_data.get('thumb'),
+        }
 
-        return info
+        entries = []
+        parts = cfg.get('videos')
+        if parts:
+            if len(parts) == 1:
+                part = str(parts[0])
+            if part:
+                smil_url = 'http://videolectures.net/%s/video/%s/smil.xml' % (lecture_slug, part)
+                smil = self._download_smil(smil_url, lecture_id)
+                info = self._parse_smil(smil, smil_url, lecture_id)
+                info['id'] = '%s_part%s' % (lecture_id, part)
+                switch = smil.find('.//switch')
+                if switch is not None:
+                    info['duration'] = parse_duration(switch.attrib.get('dur'))
+                return info
+            else:
+                for part in parts:
+                    entries.append(self.url_result('http://videolectures.net/%s/video/%s' % (lecture_slug, part), 'VideoLecturesNet'))
+                lecture_info['_type'] = 'multi_video'
+        else:
+            # Probably a playlist
+            entries = [
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+            lecture_info['_type'] = 'playlist'
+
+        lecture_info['entries'] = entries
+        return lecture_info

From a06bf87a2c6009d82ec28afe566f653b3deb11bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 21:23:41 +0100
Subject: [PATCH 1800/2145] [viidea] add support for sites using viidea service

---
 youtube_dl/extractor/__init__.py              |  2 +-
 .../{videolecturesnet.py => viidea.py}        | 33 ++++++++++++++-----
 2 files changed, 26 insertions(+), 9 deletions(-)
 rename youtube_dl/extractor/{videolecturesnet.py => viidea.py} (77%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94150a28f..0a90da73c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -724,7 +724,6 @@ from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
-from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
@@ -734,6 +733,7 @@ from .vidme import VidmeIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
+from .viidea import ViideaIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/viidea.py
similarity index 77%
rename from youtube_dl/extractor/videolecturesnet.py
rename to youtube_dl/extractor/viidea.py
index 351706362..71fb298e6 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/viidea.py
@@ -15,9 +15,23 @@ from ..utils import (
 )
 
 
-class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'
-    IE_NAME = 'videolectures.net'
+class ViideaIE(InfoExtractor):
+    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+            videolectures\.net|
+            flexilearn\.viidea\.net|
+            presentations\.ocwconsortium\.org|
+            video\.travel-zoom\.si|
+            video\.pomp-forum\.si|
+            tv\.nil\.si|
+            video\.hekovnik.com|
+            video\.szko\.si|
+            kpk\.viidea\.com|
+            inside\.viidea\.net|
+            video\.kiberpipa\.org|
+            bvvideo\.si|
+            kongres\.viidea\.net|
+            edemokracija\.viidea\.com
+        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'''
 
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
@@ -87,7 +101,9 @@ class VideoLecturesNetIE(InfoExtractor):
 
         lecture_id = str(cfg['obj_id'])
 
-        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (self._proto_relative_url(cfg['livepipe'], 'http:'), lecture_id), lecture_id)['lecture'][0]
+        base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
+
+        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
 
         lecture_info = {
             'id': lecture_id,
@@ -104,7 +120,7 @@ class VideoLecturesNetIE(InfoExtractor):
             if len(parts) == 1:
                 part = str(parts[0])
             if part:
-                smil_url = 'http://videolectures.net/%s/video/%s/smil.xml' % (lecture_slug, part)
+                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
                 smil = self._download_smil(smil_url, lecture_id)
                 info = self._parse_smil(smil, smil_url, lecture_id)
                 info['id'] = '%s_part%s' % (lecture_id, part)
@@ -114,13 +130,14 @@ class VideoLecturesNetIE(InfoExtractor):
                 return info
             else:
                 for part in parts:
-                    entries.append(self.url_result('http://videolectures.net/%s/video/%s' % (lecture_slug, part), 'VideoLecturesNet'))
+                    entries.append(self.url_result('%s/video/%s' % (base_url, lecture_id, part), 'Viidea'))
                 lecture_info['_type'] = 'multi_video'
         else:
             # Probably a playlist
+            playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
             entries = [
-                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
             lecture_info['_type'] = 'playlist'
 
         lecture_info['entries'] = entries

From 8e3a2bd6200660f9fb9d485b1c924fa5462bd566 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 17:43:23 +0100
Subject: [PATCH 1801/2145] [viidea] fix _VALID_URL regex and tests

---
 youtube_dl/extractor/viidea.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 71fb298e6..ae9a42737 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -31,7 +31,7 @@ class ViideaIE(InfoExtractor):
             bvvideo\.si|
             kongres\.viidea\.net|
             edemokracija\.viidea\.com
-        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'''
+        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?/*(?:[#?].*)?$'''
 
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
@@ -130,7 +130,7 @@ class ViideaIE(InfoExtractor):
                 return info
             else:
                 for part in parts:
-                    entries.append(self.url_result('%s/video/%s' % (base_url, lecture_id, part), 'Viidea'))
+                    entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
                 lecture_info['_type'] = 'multi_video'
         else:
             # Probably a playlist

From 6fdb39ded15c6276b49fa67cb517bf1fed63af35 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 20:38:33 +0100
Subject: [PATCH 1802/2145] [viidia] Cleaup

[viidea] extract playlist if lecture is an event

[viidia] use compat_str
---
 youtube_dl/extractor/viidea.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index ae9a42737..2541a36ed 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -4,11 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_HTTPError,
     compat_urlparse,
+    compat_str,
 )
 from ..utils import (
-    ExtractorError,
     parse_duration,
     js_to_json,
     parse_iso8601,
@@ -97,9 +96,9 @@ class ViideaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, lecture_slug)
 
-        cfg = self._parse_json(self._search_regex(r'cfg\s*:\s*({[^}]+})', webpage, 'cfg'), lecture_slug, js_to_json)
+        cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
 
-        lecture_id = str(cfg['obj_id'])
+        lecture_id = compat_str(cfg['obj_id'])
 
         base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
 
@@ -118,7 +117,7 @@ class ViideaIE(InfoExtractor):
         parts = cfg.get('videos')
         if parts:
             if len(parts) == 1:
-                part = str(parts[0])
+                part = compat_str(parts[0])
             if part:
                 smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
                 smil = self._download_smil(smil_url, lecture_id)
@@ -132,7 +131,7 @@ class ViideaIE(InfoExtractor):
                 for part in parts:
                     entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
                 lecture_info['_type'] = 'multi_video'
-        else:
+        if not parts or lecture_data.get('type') == 'evt':
             # Probably a playlist
             playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
             entries = [

From e8ce2375e0851e65c4882002297404825fe1045e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 06:54:27 +0600
Subject: [PATCH 1803/2145] [viidea] Improve and cleanup (Closes #7390)

* Optimize requests for multipart videos
* Fix cfg regex
* Improve titles and identifiers
---
 youtube_dl/extractor/viidea.py | 99 ++++++++++++++++++++++++----------
 1 file changed, 72 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 2541a36ed..525e303d4 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -35,35 +35,42 @@ class ViideaIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
-            'id': '20171_part1',
+            'id': '20171',
+            'display_id': 'promogram_igor_mekjavic_eng',
             'ext': 'mp4',
             'title': 'Automatics, robotics and biocybernetics',
             'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+            'thumbnail': 're:http://.*\.jpg',
+            'timestamp': 1372349289,
             'upload_date': '20130627',
             'duration': 565,
-            'thumbnail': 're:http://.*\.jpg',
         },
     }, {
         # video with invalid direct format links (HTTP 403)
         'url': 'http://videolectures.net/russir2010_filippova_nlp/',
         'info_dict': {
-            'id': '14891_part1',
+            'id': '14891',
+            'display_id': 'russir2010_filippova_nlp',
             'ext': 'flv',
             'title': 'NLP at Google',
             'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
-            'duration': 5352,
             'thumbnail': 're:http://.*\.jpg',
+            'timestamp': 1284375600,
+            'upload_date': '20100913',
+            'duration': 5352,
         },
         'params': {
             # rtmp download
             'skip_download': True,
         },
     }, {
+        # event playlist
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {
             'id': '23181',
             'title': 'Deep Learning Summer School, Montreal 2015',
             'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+            'thumbnail': 're:http://.*\.jpg',
             'timestamp': 1438560000,
         },
         'playlist_count': 30,
@@ -72,37 +79,54 @@ class ViideaIE(InfoExtractor):
         'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
         'info_dict': {
             'id': '9737',
+            'display_id': 'mlss09uk_bishop_ibi',
             'title': 'Introduction To Bayesian Inference',
+            'thumbnail': 're:http://.*\.jpg',
             'timestamp': 1251622800,
         },
         'playlist': [{
             'info_dict': {
                 'id': '9737_part1',
+                'display_id': 'mlss09uk_bishop_ibi_part1',
                 'ext': 'wmv',
-                'title': 'Introduction To Bayesian Inference',
+                'title': 'Introduction To Bayesian Inference (Part 1)',
+                'thumbnail': 're:http://.*\.jpg',
+                'duration': 4622,
+                'timestamp': 1251622800,
+                'upload_date': '20090830',
             },
         }, {
             'info_dict': {
                 'id': '9737_part2',
+                'display_id': 'mlss09uk_bishop_ibi_part2',
                 'ext': 'wmv',
-                'title': 'Introduction To Bayesian Inference',
+                'title': 'Introduction To Bayesian Inference (Part 2)',
+                'thumbnail': 're:http://.*\.jpg',
+                'duration': 5641,
+                'timestamp': 1251622800,
+                'upload_date': '20090830',
             },
         }],
         'playlist_count': 2,
     }]
 
     def _real_extract(self, url):
-        lecture_slug, part = re.match(self._VALID_URL, url).groups()
+        lecture_slug, explicit_part_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(url, lecture_slug)
 
-        cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
+        cfg = self._parse_json(self._search_regex(
+            [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
+             r'cfg\s*:\s*({[^}]+})'],
+            webpage, 'cfg'), lecture_slug, js_to_json)
 
         lecture_id = compat_str(cfg['obj_id'])
 
         base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
 
-        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
+        lecture_data = self._download_json(
+            '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
+            lecture_id)['lecture'][0]
 
         lecture_info = {
             'id': lecture_id,
@@ -113,31 +137,52 @@ class ViideaIE(InfoExtractor):
             'thumbnail': lecture_data.get('thumb'),
         }
 
-        entries = []
-        parts = cfg.get('videos')
+        playlist_entries = []
+        lecture_type = lecture_data.get('type')
+        parts = [compat_str(video) for video in cfg.get('videos', [])]
         if parts:
-            if len(parts) == 1:
-                part = compat_str(parts[0])
-            if part:
-                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
+            multipart = len(parts) > 1
+
+            def extract_part(part_id):
+                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
                 smil = self._download_smil(smil_url, lecture_id)
                 info = self._parse_smil(smil, smil_url, lecture_id)
-                info['id'] = '%s_part%s' % (lecture_id, part)
+                info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
+                info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
+                if multipart:
+                    info['title'] += ' (Part %s)' % part_id
                 switch = smil.find('.//switch')
                 if switch is not None:
                     info['duration'] = parse_duration(switch.attrib.get('dur'))
-                return info
+                item_info = lecture_info.copy()
+                item_info.update(info)
+                return item_info
+
+            if explicit_part_id or not multipart:
+                result = extract_part(explicit_part_id or parts[0])
             else:
-                for part in parts:
-                    entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
-                lecture_info['_type'] = 'multi_video'
-        if not parts or lecture_data.get('type') == 'evt':
-            # Probably a playlist
-            playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
+                result = {
+                    '_type': 'multi_video',
+                    'entries': [extract_part(part) for part in parts],
+                }
+                result.update(lecture_info)
+
+            # Immediately return explicitly requested part or non event item
+            if explicit_part_id or lecture_type != 'evt':
+                return result
+
+            playlist_entries.append(result)
+
+        # It's probably a playlist
+        if not parts or lecture_type == 'evt':
+            playlist_webpage = self._download_webpage(
+                '%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
             entries = [
                 self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
-            lecture_info['_type'] = 'playlist'
+                for _, video_url in re.findall(
+                    r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
+            playlist_entries.extend(entries)
 
-        lecture_info['entries'] = entries
-        return lecture_info
+        playlist = self.playlist_result(playlist_entries, lecture_id)
+        playlist.update(lecture_info)
+        return playlist

From d5c181a14e08198e400932d591b47683a630c8c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 8 Nov 2015 11:49:51 +0100
Subject: [PATCH 1804/2145] [movieclips] Fix extraction (fixes #7404)

They use theplatform now.
Changed the test, because the old one seems to be georestricted.
---
 youtube_dl/extractor/movieclips.py | 77 ++++++++----------------------
 1 file changed, 19 insertions(+), 58 deletions(-)

diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index 04e17d055..e06828b55 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -1,80 +1,41 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_str,
-)
-from ..utils import (
-    ExtractorError,
-    clean_html,
+    compat_urllib_request,
 )
 
 
 class MovieClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://movieclips\.com/(?P<id>[\da-zA-Z]+)(?:-(?P<display_id>[\da-z-]+))?'
+    _VALID_URL = r'https?://(?:www.)?movieclips\.com/videos/(?P<id>[^/?#]+)'
     _TEST = {
-        'url': 'http://movieclips.com/Wy7ZU-my-week-with-marilyn-movie-do-you-love-me/',
+        'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597?autoPlay=true&playlistId=5',
         'info_dict': {
-            'id': 'Wy7ZU',
-            'display_id': 'my-week-with-marilyn-movie-do-you-love-me',
+            'id': 'pKIGmG83AqD9',
+            'display_id': 'warcraft-trailer-1-561180739597',
             'ext': 'mp4',
-            'title': 'My Week with Marilyn - Do You Love Me?',
-            'description': 'md5:e86795bd332fe3cff461e7c8dc542acb',
+            'title': 'Warcraft Trailer 1',
+            'description': 'Watch Trailer 1 from Warcraft (2016). Legendary’s WARCRAFT is a 3D epic adventure of world-colliding conflict based.',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
+        'add_ie': ['ThePlatform'],
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-        show_id = display_id or video_id
+        display_id = self._match_id(url)
 
-        config = self._download_xml(
-            'http://config.movieclips.com/player/config/%s' % video_id,
-            show_id, 'Downloading player config')
-
-        if config.find('./country-region').text == 'false':
-            raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, config.find('./region_alert').text), expected=True)
-
-        properties = config.find('./video/properties')
-        smil_file = properties.attrib['smil_file']
-
-        smil = self._download_xml(smil_file, show_id, 'Downloading SMIL')
-        base_url = smil.find('./head/meta').attrib['base']
-
-        formats = []
-        for video in smil.findall('./body/switch/video'):
-            vbr = int(video.attrib['system-bitrate']) / 1000
-            src = video.attrib['src']
-            formats.append({
-                'url': base_url,
-                'play_path': src,
-                'ext': src.split(':')[0],
-                'vbr': vbr,
-                'format_id': '%dk' % vbr,
-            })
-
-        self._sort_formats(formats)
-
-        title = '%s - %s' % (properties.attrib['clip_movie_title'], properties.attrib['clip_title'])
-        description = clean_html(compat_str(properties.attrib['clip_description']))
-        thumbnail = properties.attrib['image']
-        categories = properties.attrib['clip_categories'].split(',')
+        req = compat_urllib_request.Request(url)
+        # it doesn't work if it thinks the browser it's too old
+        req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
+        webpage = self._download_webpage(req, display_id)
+        theplatform_link = self._html_search_regex(r'src="(http://player.theplatform.com/p/.*?)"', webpage, 'theplatform link')
+        title = self._html_search_regex(r'<title[^>]*>([^>]+)-\s*\d+\s*|\s*Movieclips.com</title>', webpage, 'title')
+        description = self._html_search_meta('description', webpage)
 
         return {
-            'id': video_id,
-            'display_id': display_id,
+            '_type': 'url_transparent',
+            'url': theplatform_link,
             'title': title,
+            'display_id': display_id,
             'description': description,
-            'thumbnail': thumbnail,
-            'categories': categories,
-            'formats': formats,
         }

From 937511dfc01c3d00c35a00f78c2b6f989b4d46e3 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Sat, 7 Nov 2015 22:55:02 +0100
Subject: [PATCH 1805/2145] Added support for the RTBF OUFtivi subpage

---
 youtube_dl/extractor/rtbf.py | 41 ++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index 04a66df90..e75b45112 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,17 +9,36 @@ from ..utils import (
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'https?://www.rtbf.be/video/[^\?]+\?id=(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
-        'md5': '799f334ddf2c0a582ba80c44655be570',
-        'info_dict': {
-            'id': '1921274',
-            'ext': 'mp4',
-            'title': 'Les Diables au coeur (épisode 2)',
-            'duration': 3099,
-        }
-    }
+    _VALID_URL = r'''(?x)
+        https?://www\.rtbf\.be/
+            (?:
+                video/[^\?]+\?id=|
+                ouftivi/heros/[^&]+&videoId=
+            )
+        (?P<id>\d+)
+    '''
+    _TESTS = [
+        {
+            'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+            'md5': '799f334ddf2c0a582ba80c44655be570',
+            'info_dict': {
+                'id': '1921274',
+                'ext': 'mp4',
+                'title': 'Les Diables au coeur (épisode 2)',
+                'duration': 3099,
+            }
+        },
+        {
+            'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
+            'md5': '25aea17e949e1e0c7c41270d60d25f22',
+            'info_dict': {
+                'id': '2057442',
+                'ext': 'mp4',
+                'title': 'Scooby-Doo, myst\xe8res associ\xe9s',
+                'duration': 1279,
+            }
+        },
+    ]
 
     _QUALITIES = [
         ('mobile', 'mobile'),

From fda2717ef9d429358d5816582590d15d18f9109f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 16:56:20 +0600
Subject: [PATCH 1806/2145] [movieclips] Add coding cookie

---
 youtube_dl/extractor/movieclips.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index e06828b55..b8c43a163 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

From 114e6025b09e12bd01b5ce22bd2c43a3ef0ba460 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 17:01:45 +0600
Subject: [PATCH 1807/2145] [rtbf] Expand _VALID_URL (Closes #7402)

---
 youtube_dl/extractor/rtbf.py | 48 ++++++++++++++----------------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e75b45112..acf10e253 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,36 +9,24 @@ from ..utils import (
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-        https?://www\.rtbf\.be/
-            (?:
-                video/[^\?]+\?id=|
-                ouftivi/heros/[^&]+&videoId=
-            )
-        (?P<id>\d+)
-    '''
-    _TESTS = [
-        {
-            'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
-            'md5': '799f334ddf2c0a582ba80c44655be570',
-            'info_dict': {
-                'id': '1921274',
-                'ext': 'mp4',
-                'title': 'Les Diables au coeur (épisode 2)',
-                'duration': 3099,
-            }
-        },
-        {
-            'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
-            'md5': '25aea17e949e1e0c7c41270d60d25f22',
-            'info_dict': {
-                'id': '2057442',
-                'ext': 'mp4',
-                'title': 'Scooby-Doo, myst\xe8res associ\xe9s',
-                'duration': 1279,
-            }
-        },
-    ]
+    _VALID_URL = r'https?://www\.rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+        'md5': '799f334ddf2c0a582ba80c44655be570',
+        'info_dict': {
+            'id': '1921274',
+            'ext': 'mp4',
+            'title': 'Les Diables au coeur (épisode 2)',
+            'duration': 3099,
+        }
+    }, {
+        # geo restricted
+        'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.rtbf.be/ouftivi/niouzz?videoId=2055858',
+        'only_matching': True,
+    }]
 
     _QUALITIES = [
         ('mobile', 'mobile'),

From aa8d2d5be6a99542b85a85af3310fab1bf641e86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 17:03:21 +0600
Subject: [PATCH 1808/2145] [rtbf] Make www optional in _VALID_URL

---
 youtube_dl/extractor/rtbf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index acf10e253..e42b319a3 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
         'md5': '799f334ddf2c0a582ba80c44655be570',

From 50506cb60798fe4d2ebb9603798b3fb1cb81e55f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:01:37 +0600
Subject: [PATCH 1809/2145] [extremetube] Fix extraction (Closes #7163)

---
 youtube_dl/extractor/extremetube.py | 45 +++++++++++++++++++----------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index c826a5404..3e11e3299 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,12 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
 from ..utils import (
-    qualities,
+    int_or_none,
     str_to_int,
 )
 
@@ -49,20 +46,36 @@ class ExtremeTubeIE(InfoExtractor):
             r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
             webpage, 'view count', fatal=False))
 
-        flash_vars = compat_parse_qs(self._search_regex(
-            r'<param[^>]+?name="flashvars"[^>]+?value="([^"]+)"', webpage, 'flash vars'))
+        flash_vars = self._parse_json(
+            self._search_regex(
+                r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flash vars'),
+            video_id)
 
         formats = []
-        quality = qualities(['180p', '240p', '360p', '480p', '720p', '1080p'])
-        for k, vals in flash_vars.items():
-            m = re.match(r'quality_(?P<quality>[0-9]+p)$', k)
-            if m is not None:
-                formats.append({
-                    'format_id': m.group('quality'),
-                    'quality': quality(m.group('quality')),
-                    'url': vals[0],
+        for quality_key, video_url in flash_vars.items():
+            height = int_or_none(self._search_regex(
+                r'quality_(\d+)[pP]$', quality_key, 'height', default=None))
+            if not height:
+                continue
+            f = {
+                'url': video_url,
+            }
+            mobj = re.search(
+                r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+            if mobj:
+                height = int(mobj.group('height'))
+                bitrate = int(mobj.group('bitrate'))
+                f.update({
+                    'format_id': '%dp-%dk' % (height, bitrate),
+                    'height': height,
+                    'tbr': bitrate,
                 })
-
+            else:
+                f.update({
+                    'format_id': '%dp' % height,
+                    'height': height,
+                })
+            formats.append(f)
         self._sort_formats(formats)
 
         return {

From cc8034cc4c52fcbfaf9f8edf34d562c481860193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:14:39 +0600
Subject: [PATCH 1810/2145] [extremetube] Modernize

---
 youtube_dl/extractor/extremetube.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 3e11e3299..c5677c82b 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -11,12 +11,12 @@ from ..utils import (
 
 
 class ExtremeTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
         'md5': '344d0c6d50e2f16b06e49ca011d8ac69',
         'info_dict': {
-            'id': '652431',
+            'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
             'ext': 'mp4',
             'title': 'Music Video 14 british euro brit european cumshots swallow',
             'uploader': 'unknown',
@@ -26,12 +26,16 @@ class ExtremeTubeIE(InfoExtractor):
     }, {
         'url': 'http://www.extremetube.com/gay/video/abcde-1234',
         'only_matching': True,
+    }, {
+        'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.extremetube.com/video/652431',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.' + mobj.group('url')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')

From f09a767d3198823e5c0ac187a91284c8d2736eb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:19:13 +0600
Subject: [PATCH 1811/2145] [mit] Allow external embeds (Closes #7406)

---
 youtube_dl/extractor/mit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index f088ab9e2..29ca45778 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -86,7 +86,7 @@ class MITIE(TechTVMITIE):
         webpage = self._download_webpage(url, page_title)
         embed_url = self._search_regex(
             r'<iframe .*?src="(.+?)"', webpage, 'embed url')
-        return self.url_result(embed_url, ie='TechTVMIT')
+        return self.url_result(embed_url)
 
 
 class OCWMITIE(InfoExtractor):

From 4f5cdf7c9b21ba00822ad0134c5883f6ca366674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 01:48:46 +0600
Subject: [PATCH 1812/2145] [cmt] Extend _VALID_URL to support shows (Closes
 #7407)

---
 youtube_dl/extractor/cmt.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index e96c59f71..f1311b14f 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -4,7 +4,7 @@ from .mtv import MTVIE
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://www\.cmt\.com/videos/.+?/(?P<videoid>[^/]+)\.jhtml'
+    _VALID_URL = r'https?://www\.cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
     _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
 
     _TESTS = [{
@@ -16,4 +16,7 @@ class CMTIE(MTVIE):
             'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
             'description': 'Blame It All On My Roots',
         },
+    }, {
+        'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
+        'only_matching': True,
     }]

From 1c31a5b0e029ae47415f7df7bea30e2b2b5df64f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:49:06 +0600
Subject: [PATCH 1813/2145] [generic] Improve kaltura embed detection (Closes
 #7409)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ee5419f51..5f61ab26f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1673,7 +1673,7 @@ class GenericIE(InfoExtractor):
 
         # Look for Kaltura embeds
         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
-                re.search(r'(?s)(["\'])(?:https?:)?//cdnapisec\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?\1.*?entry_id\s*:\s*(["\'])(?P<id>[^\2]+?)\2', webpage))
+                re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 

From 6a5d6de1e36f5278b48b7dcc92a8e15adbe541ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:49:42 +0600
Subject: [PATCH 1814/2145] [generic] Improve kaltura embed detection (2)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5f61ab26f..d0b486d2a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1672,7 +1672,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
+        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')

From b25f7533975d7463cc772e9d106802ef29252796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:50:43 +0600
Subject: [PATCH 1815/2145] [kaltura] Relax _VALID_URL

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 3dca0e566..0dcd6cd05 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -16,7 +16,7 @@ class KalturaIE(InfoExtractor):
                 (?:
                     kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
                     https?://
-                        (:?(?:www|cdnapisec)\.)?kaltura\.com/
+                        (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
                         (?:
                             (?:
                                 # flash player

From ff29bf81f852941cd7bcef1c051c77eccd4857bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Nov 2015 12:54:02 +0800
Subject: [PATCH 1816/2145] [jsinterp] Support alternative function definition
 form

---
 test/test_jsinterp.py  | 3 +++
 youtube_dl/jsinterp.py | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index fc73e5dc2..63c350b8f 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -19,6 +19,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
 
+        jsi = JSInterpreter('var x5 = function(){return 42;}')
+        self.assertEqual(jsi.call_function('x5'), 42)
+
     def test_calc(self):
         jsi = JSInterpreter('function x4(a){return 2*a+1;}')
         self.assertEqual(jsi.call_function('x4', 3), 7)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 0e0c7d90d..9bc855144 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -232,10 +232,10 @@ class JSInterpreter(object):
     def extract_function(self, funcname):
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;]%s\s*=\s*function)\s*
+                (?:function\s+%s|[{;]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
                 \{(?P<code>[^}]+)\}''' % (
-                re.escape(funcname), re.escape(funcname)),
+                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
             self.code)
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)

From 50f84a9ae171233c08ada41e03f6555c5ed95236 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Nov 2015 12:55:01 +0800
Subject: [PATCH 1817/2145] [youtube] Support new base.js html5 player

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e2a43299f..687e0b4db 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -703,7 +703,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)
@@ -1343,7 +1343,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 player_desc = 'flash player %s' % player_version
                             else:
                                 player_version = self._search_regex(
-                                    r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
+                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From 37ca7b22b5521d7797ffefcd1456b4eb31320460 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Nov 2015 11:39:06 +0100
Subject: [PATCH 1818/2145] release 2015.11.10

---
 docs/supportedsites.md | 4 +++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 805af14a0..a9820c1f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -123,6 +123,7 @@
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
+ - **democracynow**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Discovery**
  - **Dotsub**
@@ -195,6 +196,7 @@
  - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
+ - **GloboArticle**
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
@@ -617,7 +619,6 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **videolectures.net**
  - **VideoMega**
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube
@@ -627,6 +628,7 @@
  - **vier**
  - **vier:videos**
  - **Viewster**
+ - **Viidea**
  - **viki**
  - **viki:channel**
  - **vimeo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6ef482b78..b3d254005 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.02'
+__version__ = '2015.11.10'

From d3d3e2e3aac13c3c6fbae0fcfeaa41ce5ee9144b Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Tue, 10 Nov 2015 16:31:31 +0200
Subject: [PATCH 1819/2145] Adding proxy to update procedure

---
 youtube_dl/YoutubeDL.py | 3 +++
 youtube_dl/__init__.py  | 2 +-
 youtube_dl/update.py    | 7 ++-----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..5a0cc3f9a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,6 +1994,9 @@ class YoutubeDL(object):
             encoding = preferredencoding()
         return encoding
 
+    def get_opener(self):
+        return self._opener
+
     def _write_thumbnails(self, info_dict, filename):
         if self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5e2ed4d4b..760128546 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -377,7 +377,7 @@ def _real_main(argv=None):
     with YoutubeDL(ydl_opts) as ydl:
         # Update version
         if opts.update_self:
-            update_self(ydl.to_screen, opts.verbose)
+            update_self(ydl.to_screen, opts.verbose, ydl.get_opener())
 
         # Remove cache dir
         if opts.rm_cachedir:
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index fc7ac8305..04bf0939e 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -13,7 +13,7 @@ from .compat import (
     compat_str,
     compat_urllib_request,
 )
-from .utils import make_HTTPS_handler
+
 from .version import __version__
 
 
@@ -47,7 +47,7 @@ def rsa_verify(message, signature, key):
     return True
 
 
-def update_self(to_screen, verbose):
+def update_self(to_screen, verbose, opener):
     """Update the program file with the latest version from the repository"""
 
     UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
@@ -59,9 +59,6 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
-    https_handler = make_HTTPS_handler({})
-    opener = compat_urllib_request.build_opener(https_handler)
-
     # Check if there is a new version
     try:
         newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()

From 90bb5667bf32574caa7c01f66b3ca8ad2ab77e81 Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Tue, 10 Nov 2015 17:15:23 +0200
Subject: [PATCH 1820/2145] Using internal opener

---
 youtube_dl/YoutubeDL.py | 3 ---
 youtube_dl/__init__.py  | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a0cc3f9a..1783ce01b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,9 +1994,6 @@ class YoutubeDL(object):
             encoding = preferredencoding()
         return encoding
 
-    def get_opener(self):
-        return self._opener
-
     def _write_thumbnails(self, info_dict, filename):
         if self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 760128546..9f131f5db 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -377,7 +377,7 @@ def _real_main(argv=None):
     with YoutubeDL(ydl_opts) as ydl:
         # Update version
         if opts.update_self:
-            update_self(ydl.to_screen, opts.verbose, ydl.get_opener())
+            update_self(ydl.to_screen, opts.verbose, ydl._opener)
 
         # Remove cache dir
         if opts.rm_cachedir:

From 9b738b2caa20e13d0a56d05aa56af44be75476bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Nov 2015 21:32:54 +0600
Subject: [PATCH 1821/2145] [funnyordie] Fix extraction and extract m3u8
 formats

---
 youtube_dl/extractor/funnyordie.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index f5f13689c..7f21d7410 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -45,11 +45,20 @@ class FunnyOrDieIE(InfoExtractor):
 
         links.sort(key=lambda link: 1 if link[1] == 'mp4' else 0)
 
-        bitrates = self._html_search_regex(r'<source src="[^"]+/v,((?:\d+,)+)\.mp4\.csmil', webpage, 'video bitrates')
-        bitrates = [int(b) for b in bitrates.rstrip(',').split(',')]
-        bitrates.sort()
+        m3u8_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?/master\.m3u8)\1',
+            webpage, 'm3u8 url', default=None, group='url')
 
         formats = []
+
+        m3u8_formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+        if m3u8_formats:
+            formats.extend(m3u8_formats)
+
+        bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)[,/]', m3u8_url)]
+        bitrates.sort()
+
         for bitrate in bitrates:
             for link in links:
                 formats.append({

From a625e5654365a8a246fa5fc5f5d347adc67c5617 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Wed, 14 Oct 2015 11:11:52 +0200
Subject: [PATCH 1822/2145] [vidto] Add extractor

---
 docs/supportedsites.md           |  1 +
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vidto.py    | 82 ++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+)
 create mode 100644 youtube_dl/extractor/vidto.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9820c1f5..510d4d627 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -624,6 +624,7 @@
  - **VideoTt**: video.tt - Your True Tube
  - **videoweed**: VideoWeed
  - **Vidme**
+ - **vidto**: VidTo.me
  - **Vidzi**
  - **vier**
  - **vier:videos**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a90da73c..ef6c7f0de 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -730,6 +730,7 @@ from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
+from .vidto import VidtoIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
new file mode 100644
index 000000000..3cc585471
--- /dev/null
+++ b/youtube_dl/extractor/vidto.py
@@ -0,0 +1,82 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import sys
+from .common import InfoExtractor
+import time
+
+from ..utils import (
+    encode_dict,
+)
+from ..compat import (
+    compat_chr,
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlparse,
+    compat_urllib_request,
+    compat_urlparse,
+    compat_str,
+)
+
+
+class VidtoIE(InfoExtractor):
+    IE_NAME = 'vidto'
+    IE_DESC = 'VidTo.me'
+    _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
+    _HOST = 'vidto.me'
+    _TEST = {
+        'url': 'http://vidto.me/ku5glz52nqe1.html',
+        'info_dict': {
+            'id': 'ku5glz52nqe1',
+            'ext': 'mp4',
+            'title': 'test.mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        page = self._download_webpage(
+            'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
+        hash_regex = r'<input type="hidden" name="hash" value="(.*)">'
+        hash_value = self._search_regex(hash_regex, page, 'hash', fatal=True)
+        title_regex = r'<input type="hidden" name="fname" value="(.*)">'
+        title = self._search_regex(title_regex, page, 'title', fatal=False)
+        id_regex = r'<input type="hidden" name="id" value="(.*)">'
+        id_value = self._search_regex(id_regex, page, 'id', fatal=True)
+        cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
+
+
+        form_str = {
+            'op': 'download1',
+            'imhuman': 'Proceed to video',
+            'usr_login': '',
+            'id': id_value,
+            'fname': title,
+            'referer': '',
+            'hash': hash_value,
+        }
+        post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
+        req = compat_urllib_request.Request(url, post_data)
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+        for key, morsel in cookies.iteritems():
+            req.add_header('Cookie', '%s=%s' % (morsel.key, morsel.value))
+
+        print("Waiting for countdown...")
+        time.sleep(7)
+        post_result = self._download_webpage(
+            req, None,
+            note='Proceed to video...', errnote='unable to proceed', fatal=True)
+
+        file_link_regex = r'file_link ?= ?\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
+        file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
+
+        return {
+            'id': video_id,
+            'url': file_link,
+            'title': title,
+        }

From 42fc93c70928d34d46ceda644eb870e86daa8393 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Sat, 17 Oct 2015 19:35:41 +0200
Subject: [PATCH 1823/2145] vidto extractor: code cleanup

---
 docs/supportedsites.md        |  1 -
 youtube_dl/extractor/vidto.py | 32 ++++++++++++--------------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 510d4d627..a9820c1f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -624,7 +624,6 @@
  - **VideoTt**: video.tt - Your True Tube
  - **videoweed**: VideoWeed
  - **Vidme**
- - **vidto**: VidTo.me
  - **Vidzi**
  - **vier**
  - **vier:videos**
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 3cc585471..391f400fc 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -1,24 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import sys
 from .common import InfoExtractor
+import re
 import time
 
-from ..utils import (
-    encode_dict,
-)
+from ..utils import encode_dict
 from ..compat import (
-    compat_chr,
-    compat_parse_qs,
-    compat_urllib_parse,
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_unquote_plus,
-    compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urlparse,
-    compat_str,
+    compat_urllib_parse
 )
 
 
@@ -37,8 +27,7 @@ class VidtoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         page = self._download_webpage(
             'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
@@ -63,16 +52,19 @@ class VidtoIE(InfoExtractor):
         post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
         req = compat_urllib_request.Request(url, post_data)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
-        for key, morsel in cookies.iteritems():
-            req.add_header('Cookie', '%s=%s' % (morsel.key, morsel.value))
+        cookie_string = ""
+        for key in cookies.keys():
+            cookie_string += "%s=%s;" % (key, cookies[key].value)
 
-        print("Waiting for countdown...")
+        req.add_header('Cookie', '%s' % cookie_string)
+
+        self.to_screen("Waiting for countdown...")
         time.sleep(7)
         post_result = self._download_webpage(
-            req, None,
+            req, video_id,
             note='Proceed to video...', errnote='unable to proceed', fatal=True)
 
-        file_link_regex = r'file_link ?= ?\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
+        file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
         file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
 
         return {

From 37120974dc7e3e7febd40b652fc7f9e1748e2ad3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 02:02:46 +0800
Subject: [PATCH 1824/2145] [vidto] PEP8

---
 youtube_dl/extractor/vidto.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 391f400fc..d10002534 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-import re
 import time
 
 from ..utils import encode_dict
@@ -39,7 +38,6 @@ class VidtoIE(InfoExtractor):
         id_value = self._search_regex(id_regex, page, 'id', fatal=True)
         cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
 
-
         form_str = {
             'op': 'download1',
             'imhuman': 'Proceed to video',

From d8b7e80d29a67835555a6317b2c528b911077d47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= <remy.leone@gmail.com>
Date: Wed, 11 Nov 2015 12:00:31 +0100
Subject: [PATCH 1825/2145] Remove duplicate key

---
 youtube_dl/extractor/wsj.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 2ddf29a69..5a897371d 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -84,6 +84,5 @@ class WSJIE(InfoExtractor):
             'duration': duration,
             'upload_date': upload_date,
             'title': title,
-            'formats': formats,
             'categories': categories,
         }

From 990e6e8fa346224967f123a50146377088acbd75 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 20:13:03 +0800
Subject: [PATCH 1826/2145] [vidto] Minor fixes

1. import order
2. fatal is already True in helper functions
---
 youtube_dl/extractor/vidto.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index d10002534..45b21e2a9 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -1,9 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import time
 
+from .common import InfoExtractor
 from ..utils import encode_dict
 from ..compat import (
     compat_urllib_request,
@@ -60,10 +60,10 @@ class VidtoIE(InfoExtractor):
         time.sleep(7)
         post_result = self._download_webpage(
             req, video_id,
-            note='Proceed to video...', errnote='unable to proceed', fatal=True)
+            note='Proceed to video...', errnote='unable to proceed')
 
         file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
-        file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
+        file_link = self._search_regex(file_link_regex, post_result, 'file_link')
 
         return {
             'id': video_id,

From 6abce58a1226a5c359e4eb0cb228353fa7cb2aaf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 20:16:18 +0800
Subject: [PATCH 1827/2145] Credit @sieben for fixing wsj extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cc552bcb2..4cdbf4a5f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -144,3 +144,4 @@ Lee Jenkins
 Anssi Hannula
 Lukáš Lalinský
 Qijiang Fan
+Rémy Léone

From 2eb99a4b98ecece86a8d0c692066644a855f8524 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:00:23 +0600
Subject: [PATCH 1828/2145] [nowvideo] Replace main host to resolvable one

---
 youtube_dl/extractor/nowvideo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index 17baa9679..57ee3d366 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,9 +7,9 @@ class NowVideoIE(NovaMovIE):
     IE_NAME = 'nowvideo'
     IE_DESC = 'NowVideo'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|ec|sx|eu|at|ag|co|li)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
 
-    _HOST = 'www.nowvideo.ch'
+    _HOST = 'www.nowvideo.to'
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
     _FILEKEY_REGEX = r'var fkzd="([^"]+)";'

From 82393e2bb2c499cad285beb07c222f501302d830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:02:05 +0600
Subject: [PATCH 1829/2145] [novamov] Follow continue-to-the-video button if
 any (Closes #7330)

---
 youtube_dl/extractor/novamov.py | 38 +++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 04d779890..e0bf6d1bc 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -4,10 +4,14 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    NO_DEFAULT,
+    encode_dict,
+    urlencode_postdata,
 )
 
 
@@ -41,16 +45,38 @@ class NovaMovIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        page = self._download_webpage(
-            'http://%s/video/%s' % (self._HOST, video_id), video_id, 'Downloading video page')
+        url = 'http://%s/video/%s' % (self._HOST, video_id)
 
-        if re.search(self._FILE_DELETED_REGEX, page) is not None:
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading video page')
+
+        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        filekey = self._search_regex(self._FILEKEY_REGEX, page, 'filekey')
+        def extract_filekey(default=NO_DEFAULT):
+            return self._search_regex(
+                self._FILEKEY_REGEX, webpage, 'filekey', default=default)
 
-        title = self._html_search_regex(self._TITLE_REGEX, page, 'title', fatal=False)
-        description = self._html_search_regex(self._DESCRIPTION_REGEX, page, 'description', default='', fatal=False)
+        filekey = extract_filekey(default=None)
+
+        if not filekey:
+            fields = self._hidden_inputs(webpage)
+            post_url = self._search_regex(
+                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', webpage,
+                'post url', default=url, group='url')
+            if not post_url.startswith('http'):
+                post_url = compat_urlparse.urljoin(url, post_url)
+            request = compat_urllib_request.Request(
+                post_url, urlencode_postdata(encode_dict(fields)))
+            request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+            request.add_header('Referer', post_url)
+            webpage = self._download_webpage(
+                request, video_id, 'Downloading continue to the video page')
+
+        filekey = extract_filekey()
+
+        title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title', fatal=False)
+        description = self._html_search_regex(self._DESCRIPTION_REGEX, webpage, 'description', default='', fatal=False)
 
         api_response = self._download_webpage(
             'http://%s/api/player.api.php?key=%s&file=%s' % (self._HOST, filekey, video_id), video_id,

From 8b8a39e279b02926197230024e64a50d7995b95d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 23:17:59 +0800
Subject: [PATCH 1830/2145] [vidto] Several simplifications and improvements

1. Use InfoExtractor._hidden_inputs
2. Fetch title from <title> tag
3. Cookies are preserved automatically
4. Use single quotes everywhere
5. Do not declare variables for one-time use only
---
 youtube_dl/extractor/vidto.py | 51 +++++++++++++----------------------
 1 file changed, 18 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 45b21e2a9..5ebf0f3d3 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -4,24 +4,24 @@ from __future__ import unicode_literals
 import time
 
 from .common import InfoExtractor
-from ..utils import encode_dict
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse
+from ..utils import (
+    encode_dict,
+    remove_end,
+    urlencode_postdata,
 )
+from ..compat import compat_urllib_request
 
 
 class VidtoIE(InfoExtractor):
     IE_NAME = 'vidto'
     IE_DESC = 'VidTo.me'
     _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
-    _HOST = 'vidto.me'
     _TEST = {
         'url': 'http://vidto.me/ku5glz52nqe1.html',
         'info_dict': {
             'id': 'ku5glz52nqe1',
             'ext': 'mp4',
-            'title': 'test.mp4'
+            'title': 'test'
         }
     }
 
@@ -29,41 +29,26 @@ class VidtoIE(InfoExtractor):
         video_id = self._match_id(url)
 
         page = self._download_webpage(
-            'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
-        hash_regex = r'<input type="hidden" name="hash" value="(.*)">'
-        hash_value = self._search_regex(hash_regex, page, 'hash', fatal=True)
-        title_regex = r'<input type="hidden" name="fname" value="(.*)">'
-        title = self._search_regex(title_regex, page, 'title', fatal=False)
-        id_regex = r'<input type="hidden" name="id" value="(.*)">'
-        id_value = self._search_regex(id_regex, page, 'id', fatal=True)
-        cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
+            'http://vidto.me/%s.html' % video_id, video_id, 'Downloading video page')
 
-        form_str = {
-            'op': 'download1',
-            'imhuman': 'Proceed to video',
-            'usr_login': '',
-            'id': id_value,
-            'fname': title,
-            'referer': '',
-            'hash': hash_value,
-        }
-        post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
-        req = compat_urllib_request.Request(url, post_data)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-        cookie_string = ""
-        for key in cookies.keys():
-            cookie_string += "%s=%s;" % (key, cookies[key].value)
+        title = remove_end(self._html_search_regex(
+            r'<Title>\s*([^<]+)\s*</Title>', page, 'title'), ' - Vidto')
 
-        req.add_header('Cookie', '%s' % cookie_string)
+        hidden_fields = self._hidden_inputs(page)
 
-        self.to_screen("Waiting for countdown...")
+        self.to_screen('Waiting for countdown...')
         time.sleep(7)
+
+        req = compat_urllib_request.Request(
+            url, urlencode_postdata(encode_dict(hidden_fields)))
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
         post_result = self._download_webpage(
             req, video_id,
             note='Proceed to video...', errnote='unable to proceed')
 
-        file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
-        file_link = self._search_regex(file_link_regex, post_result, 'file_link')
+        file_link = self._search_regex(
+            r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)', post_result, 'file_link')
 
         return {
             'id': video_id,

From 3d9c4bf09a57871f0030d6b6022b863d66616b8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:21:21 +0600
Subject: [PATCH 1831/2145] [vimeo] Fix password protected videos (Closes
 #7451)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index ca716c8f5..9dc6247e8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -217,7 +217,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_test2=1; vuid=%s' % vuid)
+        password_request.add_header('Cookie', 'clip_test_v2=0; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,

From 9a8a12b7d8c09c6034a3d44ee64eef402c9bc076 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:23:23 +0600
Subject: [PATCH 1832/2145] [vimeo] Append cookies instead of overriding

---
 youtube_dl/extractor/vimeo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9dc6247e8..abfd9b6c4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -49,8 +49,8 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         }))
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        login_request.add_header('Cookie', 'vuid=%s' % vuid)
         login_request.add_header('Referer', self._LOGIN_URL)
+        self._set_cookie('vimeo.com', 'vuid', vuid)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
     def _extract_xsrft_and_vuid(self, webpage):
@@ -217,8 +217,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_test_v2=0; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
+        self._set_cookie('vimeo.com', 'vuid', vuid)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')
@@ -494,7 +494,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'vuid=%s' % vuid)
+        self._set_cookie('vimeo.com', 'vuid', vuid)
         self._set_cookie('vimeo.com', 'xsrft', token)
 
         return self._download_webpage(

From 9eab37dca0e767a1c9a9d659dd4125ce68d049f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:32:13 +0600
Subject: [PATCH 1833/2145] [vimeo] Simplify set cookie

---
 youtube_dl/extractor/vimeo.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index abfd9b6c4..b056ba720 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -50,7 +50,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
-        self._set_cookie('vimeo.com', 'vuid', vuid)
+        self._set_vimeo_cookie('vuid', vuid)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
     def _extract_xsrft_and_vuid(self, webpage):
@@ -62,6 +62,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             webpage, 'vuid', group='vuid')
         return xsrft, vuid
 
+    def _set_vimeo_cookie(self, name, value):
+        self._set_cookie('vimeo.com', name, value)
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -218,7 +221,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Referer', url)
-        self._set_cookie('vimeo.com', 'vuid', vuid)
+        self._set_vimeo_cookie('vuid', vuid)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')
@@ -494,8 +497,8 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
-        self._set_cookie('vimeo.com', 'vuid', vuid)
-        self._set_cookie('vimeo.com', 'xsrft', token)
+        self._set_vimeo_cookie('vuid', vuid)
+        self._set_vimeo_cookie('xsrft', token)
 
         return self._download_webpage(
             password_request, list_id,

From 699ed30ceed8a0499d17c54742b63469e5ff08ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:34:49 +0600
Subject: [PATCH 1834/2145] [novamov] Modernize

---
 youtube_dl/extractor/novamov.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index e0bf6d1bc..6b15fc2e5 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -42,8 +42,7 @@ class NovaMovIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         url = 'http://%s/video/%s' % (self._HOST, video_id)
 

From 435911029f1dfbe0caaf460a09742e864e656478 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:43:17 +0600
Subject: [PATCH 1835/2145] [vidto] Remove extractor

---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/vidto.py    | 57 --------------------------------
 2 files changed, 58 deletions(-)
 delete mode 100644 youtube_dl/extractor/vidto.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef6c7f0de..0a90da73c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -730,7 +730,6 @@ from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
-from .vidto import VidtoIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
deleted file mode 100644
index 5ebf0f3d3..000000000
--- a/youtube_dl/extractor/vidto.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import time
-
-from .common import InfoExtractor
-from ..utils import (
-    encode_dict,
-    remove_end,
-    urlencode_postdata,
-)
-from ..compat import compat_urllib_request
-
-
-class VidtoIE(InfoExtractor):
-    IE_NAME = 'vidto'
-    IE_DESC = 'VidTo.me'
-    _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
-    _TEST = {
-        'url': 'http://vidto.me/ku5glz52nqe1.html',
-        'info_dict': {
-            'id': 'ku5glz52nqe1',
-            'ext': 'mp4',
-            'title': 'test'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        page = self._download_webpage(
-            'http://vidto.me/%s.html' % video_id, video_id, 'Downloading video page')
-
-        title = remove_end(self._html_search_regex(
-            r'<Title>\s*([^<]+)\s*</Title>', page, 'title'), ' - Vidto')
-
-        hidden_fields = self._hidden_inputs(page)
-
-        self.to_screen('Waiting for countdown...')
-        time.sleep(7)
-
-        req = compat_urllib_request.Request(
-            url, urlencode_postdata(encode_dict(hidden_fields)))
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        post_result = self._download_webpage(
-            req, video_id,
-            note='Proceed to video...', errnote='unable to proceed')
-
-        file_link = self._search_regex(
-            r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)', post_result, 'file_link')
-
-        return {
-            'id': video_id,
-            'url': file_link,
-            'title': title,
-        }

From b9ad101926b928445acf4a1ab48c24e17836b22c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:44:03 +0600
Subject: [PATCH 1836/2145] [gorillavid] Add support for vidto.me

---
 youtube_dl/extractor/gorillavid.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index d23e3eac1..d4fd0449c 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -16,10 +16,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -105,12 +105,17 @@ class GorillaVidIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = self._search_regex(
-            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'<td nowrap>([^<]+)</td>', r'>Watch (.+) '],
+            [r'style="z-index: [0-9]+;">([^<]+)</span>',
+             r'<td nowrap>([^<]+)</td>',
+             r'>Watch (.+) ',
+             r'<h2 class="video-page-head">([^<]+)</h2>'],
             webpage, 'title', default=None) or self._og_search_title(webpage)
         video_url = self._search_regex(
-            r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')
+            [r'file\s*:\s*["\'](http[^"\']+)["\'],',
+             r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],
+            webpage, 'file url')
         thumbnail = self._search_regex(
-            r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', fatal=False)
+            r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)
 
         formats = [{
             'format_id': 'sd',

From 668db403f97eb1b50a3f54266db4bc274f1fa874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:47:28 +0600
Subject: [PATCH 1837/2145] [gorillavid] Add test for vidto.me and strip title

---
 youtube_dl/extractor/gorillavid.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index d4fd0449c..8c982d8e9 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -76,6 +76,13 @@ class GorillaVidIE(InfoExtractor):
             'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://vidto.me/ku5glz52nqe1.html',
+        'info_dict': {
+            'id': 'ku5glz52nqe1',
+            'ext': 'mp4',
+            'title': 'test'
+        }
     }]
 
     def _real_extract(self, url):
@@ -104,12 +111,12 @@ class GorillaVidIE(InfoExtractor):
 
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
-        title = self._search_regex(
+        title = (self._search_regex(
             [r'style="z-index: [0-9]+;">([^<]+)</span>',
              r'<td nowrap>([^<]+)</td>',
              r'>Watch (.+) ',
              r'<h2 class="video-page-head">([^<]+)</h2>'],
-            webpage, 'title', default=None) or self._og_search_title(webpage)
+            webpage, 'title', default=None) or self._og_search_title(webpage)).strip()
         video_url = self._search_regex(
             [r'file\s*:\s*["\'](http[^"\']+)["\'],',
              r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],

From 031ec536f00af5dc4ec034d3adb7d5fe3cc02453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:58:39 +0600
Subject: [PATCH 1838/2145] [gorillavid] Rename to xfileshare

---
 youtube_dl/extractor/__init__.py                      | 2 +-
 youtube_dl/extractor/{gorillavid.py => xfileshare.py} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename youtube_dl/extractor/{gorillavid.py => xfileshare.py} (96%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a90da73c..06d25ef40 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -221,7 +221,6 @@ from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
-from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
 from .groupon import GrouponIE
 from .hark import HarkIE
@@ -786,6 +785,7 @@ from .wrzuta import WrzutaIE
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
+from .xfileshare import XFileShareIE
 from .xhamster import (
     XHamsterIE,
     XHamsterEmbedIE,
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/xfileshare.py
similarity index 96%
rename from youtube_dl/extractor/gorillavid.py
rename to youtube_dl/extractor/xfileshare.py
index 8c982d8e9..7610dc627 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -15,8 +15,8 @@ from ..utils import (
 )
 
 
-class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
+class XFileShareIE(InfoExtractor):
+    IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
             (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/

From fcd817a32636268156dfab20ffc7896fb6082f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Nov 2015 03:56:11 +0600
Subject: [PATCH 1839/2145] [vimeo] Fix extraction (Closes #7460)

---
 youtube_dl/extractor/vimeo.py | 52 ++++++++++++-----------------------
 1 file changed, 17 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b056ba720..b72341a2b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -387,47 +387,29 @@ class VimeoIE(VimeoBaseInfoExtractor):
             like_count = None
             comment_count = None
 
-        # Vimeo specific: extract request signature and timestamp
-        sig = config['request']['signature']
-        timestamp = config['request']['timestamp']
-
-        # Vimeo specific: extract video codec and quality information
-        # First consider quality, then codecs, then take everything
-        codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
-        files = {'hd': [], 'sd': [], 'other': []}
-        config_files = config["video"].get("files") or config["request"].get("files")
-        for codec_name, codec_extension in codecs:
-            for quality in config_files.get(codec_name, []):
-                format_id = '-'.join((codec_name, quality)).lower()
-                key = quality if quality in files else 'other'
-                video_url = None
-                if isinstance(config_files[codec_name], dict):
-                    file_info = config_files[codec_name][quality]
-                    video_url = file_info.get('url')
-                else:
-                    file_info = {}
-                if video_url is None:
-                    video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
-                        % (video_id, sig, timestamp, quality, codec_name.upper())
-
-                files[key].append({
-                    'ext': codec_extension,
-                    'url': video_url,
-                    'format_id': format_id,
-                    'width': int_or_none(file_info.get('width')),
-                    'height': int_or_none(file_info.get('height')),
-                    'tbr': int_or_none(file_info.get('bitrate')),
-                })
         formats = []
-        m3u8_url = config_files.get('hls', {}).get('all')
+        config_files = config['video'].get('files') or config['request'].get('files', {})
+        for f in config_files.get('progressive', []):
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': 'http-%s' % f.get('quality'),
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'fps': int_or_none(f.get('fps')),
+                'tbr': int_or_none(f.get('bitrate')),
+            })
+        m3u8_url = config_files.get('hls', {}).get('url')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
-        for key in ('other', 'sd', 'hd'):
-            formats += files[key]
-        self._sort_formats(formats)
+        # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
+        # at the same time without actual units specified. This lead to wrong sorting.
+        self._sort_formats(formats, field_preference=('height', 'width', 'fps', 'format_id'))
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From cf5881fc4defe3902367fc02b8eb39d912572d5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Nov 2015 02:13:42 +0800
Subject: [PATCH 1840/2145] Credit @ferama

For providing idea for vidto.me (#7167) and extending nowvideo support (#6760)
---
 AUTHORS                         |  1 +
 youtube_dl/extractor/twitter.py | 58 +++++++++++++++++++++++++++------
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 4cdbf4a5f..f465d20ed 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -145,3 +145,4 @@ Anssi Hannula
 Lukáš Lalinský
 Qijiang Fan
 Rémy Léone
+Marco Ferragina
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 9d3e46b94..ae1212d19 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -9,6 +9,8 @@ from ..utils import (
     float_or_none,
     xpath_text,
     remove_end,
+    int_or_none,
+    ExtractorError,
 )
 
 
@@ -120,7 +122,7 @@ class TwitterIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
     _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
         'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
         'info_dict': {
@@ -133,7 +135,19 @@ class TwitterIE(InfoExtractor):
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
         },
-    }
+    }, {
+        'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
+        'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
+        'info_dict': {
+            'id': '657991469417025536',
+            'ext': 'mp4',
+            'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
+            'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
+            'thumbnail': 're:^https?://.*\.png',
+            'uploader': 'Gifs',
+            'uploader_id': 'giphz',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -150,17 +164,41 @@ class TwitterIE(InfoExtractor):
         mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
         title, short_url = mobj.groups()
 
-        card_id = self._search_regex(
-            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url')
-        card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'TwitterCard',
+        info = {
             'uploader_id': user_id,
             'uploader': username,
-            'url': card_url,
             'webpage_url': url,
             'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
             'title': username + ' - ' + title,
         }
+
+        card_id = self._search_regex(
+            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url', default=None)
+        if card_id:
+            card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'TwitterCard',
+                'url': card_url,
+            })
+            return info
+
+        mobj = re.search(r'''(?x)
+            <video[^>]+class="animated-gif"[^>]+
+                (?:data-height="(?P<height>\d+)")?[^>]+
+                (?:data-width="(?P<width>\d+)")?[^>]+
+                (?:poster="(?P<poster>[^"]+)")?[^>]*>\s*
+                <source[^>]+video-src="(?P<url>[^"]+)"
+        ''', webpage)
+
+        if mobj:
+            info.update({
+                'id': twid,
+                'url': mobj.group('url'),
+                'height': int_or_none(mobj.group('height')),
+                'width': int_or_none(mobj.group('width')),
+                'thumbnail': mobj.group('poster'),
+            })
+            return info
+
+        raise ExtractorError('There\'s not video in this tweet.')

From 1ebb4717dfc67890c66abd4274e201cafda48e79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Nov 2015 21:02:56 +0100
Subject: [PATCH 1841/2145] [cbsnews] Fix construction of 'play_path' in some
 videos (fixes #7394)

---
 youtube_dl/extractor/cbsnews.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 52e61d85b..f9a64a0a2 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -67,9 +67,12 @@ class CBSNewsIE(InfoExtractor):
                 'format_id': format_id,
             }
             if uri.startswith('rtmp'):
+                play_path = re.sub(
+                    r'{slistFilePath}', '',
+                    uri.split('<break>')[-1].split('{break}')[-1])
                 fmt.update({
                     'app': 'ondemand?auth=cbs',
-                    'play_path': 'mp4:' + uri.split('<break>')[-1],
+                    'play_path': 'mp4:' + play_path,
                     'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
                     'page_url': 'http://www.cbsnews.com',
                     'ext': 'flv',

From 5d6c3d6a665ed5de89bb3b056bb051b043400897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Nov 2015 02:41:38 +0600
Subject: [PATCH 1842/2145] [ruutu] Skip NOT-USED URLs(Closes #7478)

---
 youtube_dl/extractor/ruutu.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index a16b73ff4..c19107cb0 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -57,7 +57,8 @@ class RuutuIE(InfoExtractor):
                     extract_formats(child)
                 elif child.tag.endswith('File'):
                     video_url = child.text
-                    if not video_url or video_url in processed_urls or 'NOT_USED' in video_url:
+                    if (not video_url or video_url in processed_urls or
+                            any(p in video_url for p in ('NOT_USED', 'NOT-USED'))):
                         return
                     processed_urls.append(video_url)
                     ext = determine_ext(video_url)

From 91d644b5ba7e520363d146442682b7df5a175d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Nov 2015 02:43:27 +0600
Subject: [PATCH 1843/2145] [ruutu] Relax formats extraction

---
 youtube_dl/extractor/ruutu.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index c19107cb0..e417bf661 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -63,11 +63,15 @@ class RuutuIE(InfoExtractor):
                     processed_urls.append(video_url)
                     ext = determine_ext(video_url)
                     if ext == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            video_url, video_id, 'mp4', m3u8_id='hls'))
+                        m3u8_formats = self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
                     elif ext == 'f4m':
-                        formats.extend(self._extract_f4m_formats(
-                            video_url, video_id, f4m_id='hds'))
+                        f4m_formats = self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
                     else:
                         proto = compat_urllib_parse_urlparse(video_url).scheme
                         if not child.tag.startswith('HTTP') and proto != 'rtmp':

From a1ec9a7553b6bb09b0f88ce5e987c04e2f16ba4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Nov 2015 11:07:30 +0100
Subject: [PATCH 1844/2145] release 2015.11.13

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9820c1f5..5016ba4bc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -200,7 +200,6 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com
  - **Goshgay**
  - **Groupon**
  - **Hark**
@@ -671,6 +670,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
+ - **XFileShare**: XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me
  - **XHamster**
  - **XHamsterEmbed**
  - **XMinus**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b3d254005..6585d60d5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.10'
+__version__ = '2015.11.13'

From b84a5f03375d086e9912cfa7c690ff824b1092a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Nov 2015 18:55:07 +0100
Subject: [PATCH 1845/2145] [twitter] Update tests checksums

---
 youtube_dl/extractor/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ae1212d19..0db7a4440 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -20,7 +20,7 @@ class TwitterCardIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-            'md5': '7d2f6b4d2eb841a7ccc893d479bfceb4',
+            'md5': '4fa26a35f9d1bf4b646590ba8e84be19',
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
@@ -124,7 +124,7 @@ class TwitterIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
-        'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
+        'md5': 'db6612ec5d03355953c3ca9250c97e5e',
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',

From b703ebeeafed3e890270633788c7340d174cde86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Nov 2015 19:09:42 +0100
Subject: [PATCH 1846/2145] [twitter] Don't fail if the description doesn't
 contain an URL (fixes #7489)

---
 youtube_dl/extractor/twitter.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 0db7a4440..2bd5946ac 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -147,6 +147,17 @@ class TwitterIE(InfoExtractor):
             'uploader': 'Gifs',
             'uploader_id': 'giphz',
         },
+    }, {
+        'url': 'https://twitter.com/starwars/status/665052190608723968',
+        'md5': '39b7199856dee6cd4432e72c74bc69d4',
+        'info_dict': {
+            'id': '665052190608723968',
+            'ext': 'mp4',
+            'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
+            'description': 'Star Wars on Twitter: "A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens."',
+            'uploader_id': 'starwars',
+            'uploader': 'Star Wars',
+        },
     }]
 
     def _real_extract(self, url):
@@ -158,17 +169,16 @@ class TwitterIE(InfoExtractor):
 
         username = remove_end(self._og_search_title(webpage), ' on Twitter')
 
-        title = self._og_search_description(webpage).strip('').replace('\n', ' ')
+        title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')
 
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
-        mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
-        title, short_url = mobj.groups()
+        title = re.sub(r'\s+(https?://[^ ]+)', '', title)
 
         info = {
             'uploader_id': user_id,
             'uploader': username,
             'webpage_url': url,
-            'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
+            'description': '%s on Twitter: "%s"' % (username, description),
             'title': username + ' - ' + title,
         }
 

From 4e21b3a94f1ce7ba3757d59d1ac4baf9efaeed84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 00:22:32 +0600
Subject: [PATCH 1847/2145] [cbs] Use android UA for higher quality streams
 (Closes #7490)

---
 youtube_dl/extractor/cbs.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 75fffb156..43f05d278 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import smuggle_url
 
 
 class CBSIE(InfoExtractor):
@@ -46,13 +48,19 @@ class CBSIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        request = compat_urllib_request.Request(url)
+        # Android UA is served with higher quality (720p) streams (see
+        # https://github.com/rg3/youtube-dl/issues/7490)
+        request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
+        webpage = self._download_webpage(request, display_id)
         real_id = self._search_regex(
             [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
         return {
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
-            'url': 'theplatform:%s' % real_id,
+            'url': smuggle_url(
+                'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true&manifest=m3u' % real_id,
+                {'force_smil_url': True}),
             'display_id': display_id,
         }

From a662489877aa8d88b898a4984c2e580b9edbe7de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:09:50 +0600
Subject: [PATCH 1848/2145] [brightcove:embedinpage] Make more robust and
 extract rtmp streams

---
 youtube_dl/extractor/brightcove.py | 117 +++++++++++++++++++----------
 1 file changed, 79 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2c7d968a8..2ad35ec90 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -23,6 +23,7 @@ from ..utils import (
     unescapeHTML,
     unsmuggle_url,
     js_to_json,
+    float_or_none,
     int_or_none,
     parse_iso8601,
     extract_attributes,
@@ -353,7 +354,7 @@ class BrightcoveIE(InfoExtractor):
 
 
 class BrightcoveInPageEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[\da-f-]+)_(?P<embed>[a-z]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -385,59 +386,99 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
     def _real_extract(self, url):
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 
-        webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
+        webpage = self._download_webpage(
+            'http://players.brightcove.net/%s/%s_%s/index.min.js'
+            % (account_id, player_id, embed), video_id)
 
-        catalog = self._parse_json(
-            js_to_json(
-                self._search_regex(
-                    r'catalog\(({[^}]+})\);',
-                    webpage,
-                    'catalog'
-                )
-            ),
-            video_id
-        )
-        policy_key = catalog['policyKey']
+        policy_key = None
+
+        catalog = self._search_regex(
+            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
+        if catalog:
+            catalog = self._parse_json(
+                js_to_json(catalog), video_id, fatal=False)
+            if catalog:
+                policy_key = catalog.get('policyKey')
+
+        if not policy_key:
+            policy_key = self._search_regex(
+                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
+                webpage, 'policy key', group='pk')
 
         req = compat_urllib_request.Request(
-            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id),
+            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
+            % (account_id, video_id),
             headers={'Accept': 'application/json;pk=%s' % policy_key})
         json_data = self._download_json(req, video_id)
 
         title = json_data['name']
+
+        formats = []
+        for source in json_data.get('sources', []):
+            source_type = source.get('type')
+            src = source.get('src')
+            if source_type == 'application/x-mpegURL':
+                if not src:
+                    continue
+                m3u8_formats = self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            else:
+                streaming_src = source.get('streaming_src')
+                stream_name, app_name = source.get('stream_name'), source.get('app_name')
+                if not src and not streaming_src and (not stream_name or not app_name):
+                    continue
+                tbr = float_or_none(source.get('avg_bitrate'), 1000)
+                height = int_or_none(source.get('height'))
+                f = {
+                    'tbr': tbr,
+                    'width': int_or_none(source.get('width')),
+                    'height': height,
+                    'filesize': int_or_none(source.get('size')),
+                    'container': source.get('container'),
+                    'vcodec': source.get('codec'),
+                    'ext': source.get('container').lower(),
+                }
+
+                def build_format_id(kind):
+                    format_id = kind
+                    if tbr:
+                        format_id += '-%dk' % int(tbr)
+                    if height:
+                        format_id += '-%dp' % height
+                    return format_id
+
+                if src or streaming_src:
+                    f.update({
+                        'url': src or streaming_src,
+                        'format_id': build_format_id('http' if src else 'http-streaming'),
+                        'preference': 2 if src else 1,
+                    })
+                else:
+                    f.update({
+                        'url': app_name,
+                        'play_path': stream_name,
+                        'format_id': build_format_id('rtmp'),
+                    })
+                formats.append(f)
+        self._sort_formats(formats)
+
         description = json_data.get('description')
         thumbnail = json_data.get('thumbnail')
         timestamp = parse_iso8601(json_data.get('published_at'))
-        duration = int_or_none(json_data.get('duration'))
-
-        formats = []
-        for source in json_data.get('sources'):
-            source_type = source.get('type')
-            if source_type == 'application/x-mpegURL':
-                formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
-            else:
-                src = source.get('src') or source.get('streaming_src')
-                if src:
-                    formats.append({
-                        'url': src,
-                        'tbr': source.get('avg_bitrate'),
-                        'width': int_or_none(source.get('width')),
-                        'height': int_or_none(source.get('height')),
-                        'filesize': source.get('size'),
-                        'container': source.get('container'),
-                        'vcodec': source.get('codec'),
-                        'ext': source.get('container').lower(),
-                    })
-
-        self._sort_formats(formats)
+        duration = float_or_none(json_data.get('duration'), 1000)
+        tags = json_data.get('tags', [])
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'timestamp': timestamp,
             'duration': duration,
-            'formats': formats,
+            'timestamp': timestamp,
             'uploader_id': account_id,
+            'formats': formats,
+            'tags': tags,
         }

From 536f819eda975224666374a9ce83cc3472f5aa5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:51:05 +0600
Subject: [PATCH 1849/2145] [brightcove] Imrove extraction of new embeds

---
 youtube_dl/extractor/brightcove.py | 42 +++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2ad35ec90..d494b8b67 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -354,7 +354,7 @@ class BrightcoveIE(InfoExtractor):
 
 
 class BrightcoveInPageEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[\da-f-]+)_(?P<embed>[a-z]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -370,18 +370,34 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
         }
     }
 
-    @staticmethod
-    def _extract_url(webpage):
-        video_attributes = re.search(r'(?s)<video([^>]*)>.*?</(?:video|audio)>', webpage)
-        if video_attributes:
-            video_attributes = extract_attributes(video_attributes.group(), r'(?s)\s*data-(account|video-id|playlist-id|policy-key|player|embed)\s*=\s*["\']([^"\']+)["\']')
-            account_id = video_attributes.get('account')
-            player_id = video_attributes.get('player')
-            embed = video_attributes.get('embed')
-            video_id = video_attributes.get('video-id')
-            if account_id and player_id and embed and video_id:
-                return 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (account_id, player_id, embed, video_id)
-        return None
+    def _extract_urls(self, webpage):
+        # Reference:
+        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
+
+        entries = []
+
+        # Look for iframe embeds [1]
+        for _, url in re.findall(
+                r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
+            entries.append(self.url_result(self._proto_relative_url(url)))
+        # Look for embed_in_page embeds [2]
+        # According to examples from [3] it's unclear whether video id may be optional
+        # and what to do when it is
+        for video_id, account_id, player_id, embed in re.findall(
+                r'''(?sx)
+                    <video[^>]+
+                        data-video-id=["\'](\d+)["\'][^>]*>.*?
+                    </video>.*?
+                    <script[^>]+
+                        src=["\'](?:https?:)?//players\.brightcove\.net/
+                        (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
+                ''', webpage):
+            entries.append(self.url_result(
+                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                % (account_id, player_id, embed, video_id)))
+        return entries
 
     def _real_extract(self, url):
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()

From 4fcaa4f4a5ef328009bef53ebc491ebe76452550 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:54:16 +0600
Subject: [PATCH 1850/2145] [brightcove] Rename extractor to brightcove legacy

Old embedding approaches are now "Legacy Studio"
---
 youtube_dl/extractor/__init__.py   | 2 +-
 youtube_dl/extractor/brightcove.py | 2 +-
 youtube_dl/extractor/generic.py    | 4 ++--
 youtube_dl/extractor/nowness.py    | 4 ++--
 youtube_dl/extractor/safari.py     | 4 ++--
 youtube_dl/extractor/space.py      | 6 +++---
 youtube_dl/extractor/tlc.py        | 6 +++---
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 08cb93d76..8a0f76d7e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -61,7 +61,7 @@ from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
 from .brightcove import (
-    BrightcoveIE,
+    BrightcoveLegacyIE,
     BrightcoveInPageEmbedIE,
 )
 from .buzzfeed import BuzzFeedIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d494b8b67..4dbc2e975 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -30,7 +30,7 @@ from ..utils import (
 )
 
 
-class BrightcoveIE(InfoExtractor):
+class BrightcoveLegacyIE(InfoExtractor):
     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 34d930a2d..8f99dd9b1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -31,7 +31,7 @@ from ..utils import (
     xpath_text,
 )
 from .brightcove import (
-    BrightcoveIE,
+    BrightcoveLegacyIE,
     BrightcoveInPageEmbedIE,
 )
 from .nbc import NBCSportsVPlayerIE
@@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor):
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
         # Look for BrightCove:
-        bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
+        bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
             self.to_screen('Brightcove video detected.')
             entries = [{
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index b97f62fdb..dab487ea4 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,7 +1,7 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 from .common import InfoExtractor
 from ..utils import ExtractorError
 from ..compat import (
@@ -22,7 +22,7 @@ class NownessBaseIE(InfoExtractor):
                             'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
                             note='Downloading player JavaScript',
                             errnote='Unable to download player JavaScript')
-                        bc_url = BrightcoveIE._extract_brightcove_url(player_code)
+                        bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                         if bc_url is None:
                             raise ExtractorError('Could not find player definition')
                         return self.url_result(bc_url, 'Brightcove')
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index a602af692..4f1f05c6a 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 
 from ..compat import (
     compat_urllib_parse,
@@ -112,7 +112,7 @@ class SafariIE(SafariBaseIE):
             '%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
             part)
 
-        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+        bc_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if not bc_url:
             raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
 
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index c2d0d36a6..2f190f764 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 from ..utils import RegexNotFoundError, ExtractorError
 
 
@@ -31,8 +31,8 @@ class SpaceIE(InfoExtractor):
             brightcove_url = self._og_search_video_url(webpage)
         except RegexNotFoundError:
             # Other videos works fine with the info from the object
-            brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
+            brightcove_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if brightcove_url is None:
             raise ExtractorError(
                 'The webpage does not contain a video', expected=True)
-        return self.url_result(brightcove_url, BrightcoveIE.ie_key())
+        return self.url_result(brightcove_url, BrightcoveLegacyIE.ie_key())
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 13263614c..d6d038a8d 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 from .discovery import DiscoveryIE
 from ..compat import compat_urlparse
 
@@ -66,6 +66,6 @@ class TlcDeIE(InfoExtractor):
 
         return {
             '_type': 'url',
-            'url': BrightcoveIE._extract_brightcove_url(iframe),
-            'ie': BrightcoveIE.ie_key(),
+            'url': BrightcoveLegacyIE._extract_brightcove_url(iframe),
+            'ie': BrightcoveLegacyIE.ie_key(),
         }

From 5c17f0a67a3a0518f448825eee54d16045acd63c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:55:59 +0600
Subject: [PATCH 1851/2145] [brightcove:embedinpage] Rename extractor to
 brightcove new

It's not actually embed_in_page but "New Studio" and allows both iframe and embed_in_page embeds
---
 youtube_dl/extractor/__init__.py   | 2 +-
 youtube_dl/extractor/brightcove.py | 2 +-
 youtube_dl/extractor/generic.py    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8a0f76d7e..64ce3210b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -62,7 +62,7 @@ from .br import BRIE
 from .breakcom import BreakIE
 from .brightcove import (
     BrightcoveLegacyIE,
-    BrightcoveInPageEmbedIE,
+    BrightcoveNewIE,
 )
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4dbc2e975..5aca0b378 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -353,7 +353,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         return info
 
 
-class BrightcoveInPageEmbedIE(InfoExtractor):
+class BrightcoveNewIE(InfoExtractor):
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f99dd9b1..0797e1a90 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,7 +32,7 @@ from ..utils import (
 )
 from .brightcove import (
     BrightcoveLegacyIE,
-    BrightcoveInPageEmbedIE,
+    BrightcoveNewIE,
 )
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
@@ -1322,7 +1322,7 @@ class GenericIE(InfoExtractor):
             }
 
         # Look for Brightcove In Page Embed:
-        brightcove_in_page_embed_url = BrightcoveInPageEmbedIE._extract_url(webpage)
+        brightcove_in_page_embed_url = BrightcoveNewIE._extract_url(webpage)
         if brightcove_in_page_embed_url:
             return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
 

From e721d857c2b24c10c09626a4a79172d85e0dc5fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:56:51 +0600
Subject: [PATCH 1852/2145] [brightcove] Clarify IE_NAMEs

---
 youtube_dl/extractor/brightcove.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5aca0b378..8ee5486fe 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -31,6 +31,7 @@ from ..utils import (
 
 
 class BrightcoveLegacyIE(InfoExtractor):
+    IE_NAME = 'brightcove:legacy'
     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
@@ -354,6 +355,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
 
 class BrightcoveNewIE(InfoExtractor):
+    IE_NAME = 'brightcove:new'
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',

From 24af85298ed1862ac809677e70ff59f3e9ee3234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:01:56 +0600
Subject: [PATCH 1853/2145] [brightcove] Fix _extract_urls

---
 youtube_dl/extractor/brightcove.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 8ee5486fe..1c7783dcb 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -372,7 +372,8 @@ class BrightcoveNewIE(InfoExtractor):
         }
     }
 
-    def _extract_urls(self, webpage):
+    @staticmethod
+    def _extract_urls(webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
@@ -383,7 +384,7 @@ class BrightcoveNewIE(InfoExtractor):
         # Look for iframe embeds [1]
         for _, url in re.findall(
                 r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
-            entries.append(self.url_result(self._proto_relative_url(url)))
+            entries.append(url)
         # Look for embed_in_page embeds [2]
         # According to examples from [3] it's unclear whether video id may be optional
         # and what to do when it is
@@ -396,9 +397,9 @@ class BrightcoveNewIE(InfoExtractor):
                         src=["\'](?:https?:)?//players\.brightcove\.net/
                         (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
                 ''', webpage):
-            entries.append(self.url_result(
+            entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, embed, video_id)))
+                % (account_id, player_id, embed, video_id))
         return entries
 
     def _real_extract(self, url):

From f6519f89b09be788549f68ba12f0cc31c55d9751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:03:07 +0600
Subject: [PATCH 1854/2145] [generic] Extract Brightcove New Studio embeds

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0797e1a90..334864db3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1321,10 +1321,10 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
-        # Look for Brightcove In Page Embed:
-        brightcove_in_page_embed_url = BrightcoveNewIE._extract_url(webpage)
-        if brightcove_in_page_embed_url:
-            return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
+        # Look for Brightcove New Studio embeds
+        bc_urls = BrightcoveNewIE._extract_urls(webpage)
+        if bc_urls:
+            return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
 
         # Look for embedded rtl.nl player
         matches = re.findall(

From 1f4b722b00fd5c24468cd4d072e8b5c5428ca515 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:03:32 +0600
Subject: [PATCH 1855/2145] [generic] Clarify Brightcove Legacy Studio comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 334864db3..8ba0a9913 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1304,7 +1304,7 @@ class GenericIE(InfoExtractor):
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
-        # Look for BrightCove:
+        # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
             self.to_screen('Brightcove video detected.')

From 3b7d9aa487399e06bba5dc03c90b6576c2b067b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:05:46 +0600
Subject: [PATCH 1856/2145] Rename all references to legacy studio Brightcove
 extractor

---
 youtube_dl/extractor/aljazeera.py | 4 ++--
 youtube_dl/extractor/generic.py   | 8 ++++----
 youtube_dl/extractor/nowness.py   | 2 +-
 youtube_dl/extractor/safari.py    | 2 +-
 youtube_dl/extractor/space.py     | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 184a14a4f..5b2c0dc9a 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -15,7 +15,7 @@ class AlJazeeraIE(InfoExtractor):
             'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
             'uploader': 'Al Jazeera English',
         },
-        'add_ie': ['Brightcove'],
+        'add_ie': ['BrightcoveLegacy'],
         'skip': 'Not accessible from Travis CI server',
     }
 
@@ -32,5 +32,5 @@ class AlJazeeraIE(InfoExtractor):
                 'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
                 '&%40videoPlayer={0}'.format(brightcove_id)
             ),
-            'ie_key': 'Brightcove',
+            'ie_key': 'BrightcoveLegacy',
         }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8ba0a9913..51516a38a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -278,7 +278,7 @@ class GenericIE(InfoExtractor):
         # it also tests brightcove videos that need to set the 'Referer' in the
         # http requests
         {
-            'add_ie': ['Brightcove'],
+            'add_ie': ['BrightcoveLegacy'],
             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
             'info_dict': {
                 'id': '2765128793001',
@@ -302,7 +302,7 @@ class GenericIE(InfoExtractor):
                 'uploader': 'thestar.com',
                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
             },
-            'add_ie': ['Brightcove'],
+            'add_ie': ['BrightcoveLegacy'],
         },
         {
             'url': 'http://www.championat.com/video/football/v/87/87499.html',
@@ -317,7 +317,7 @@ class GenericIE(InfoExtractor):
         },
         {
             # https://github.com/rg3/youtube-dl/issues/3541
-            'add_ie': ['Brightcove'],
+            'add_ie': ['BrightcoveLegacy'],
             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
             'info_dict': {
                 'id': '3866516442001',
@@ -1311,7 +1311,7 @@ class GenericIE(InfoExtractor):
             entries = [{
                 '_type': 'url',
                 'url': smuggle_url(bc_url, {'Referer': url}),
-                'ie_key': 'Brightcove'
+                'ie_key': 'BrightcoveLegacy'
             } for bc_url in bc_urls]
 
             return {
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index dab487ea4..0fba55833 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -25,7 +25,7 @@ class NownessBaseIE(InfoExtractor):
                         bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                         if bc_url is None:
                             raise ExtractorError('Could not find player definition')
-                        return self.url_result(bc_url, 'Brightcove')
+                        return self.url_result(bc_url, 'BrightcoveLegacy')
                     elif source == 'vimeo':
                         return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
                     elif source == 'youtube':
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 4f1f05c6a..e9e33d0a3 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -116,7 +116,7 @@ class SafariIE(SafariBaseIE):
         if not bc_url:
             raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
 
-        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'Brightcove')
+        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'BrightcoveLegacy')
 
 
 class SafariCourseIE(SafariBaseIE):
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index 2f190f764..ebb5d6ec0 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -10,7 +10,7 @@ from ..utils import RegexNotFoundError, ExtractorError
 class SpaceIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|m)\.)?space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html'
     _TEST = {
-        'add_ie': ['Brightcove'],
+        'add_ie': ['BrightcoveLegacy'],
         'url': 'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
         'info_dict': {
             'id': '2780937028001',

From 75eac8961ee2ff004891ec57d5a2fec4f0b5574d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:07:24 +0600
Subject: [PATCH 1857/2145] [brightcove] Remove unused import

---
 youtube_dl/extractor/brightcove.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 1c7783dcb..ef34ae48f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -26,7 +26,6 @@ from ..utils import (
     float_or_none,
     int_or_none,
     parse_iso8601,
-    extract_attributes,
 )
 
 

From c7b959ce383040f1d507eef0e43041029583b307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:07:44 +0600
Subject: [PATCH 1858/2145] [utils] Remove unused function

---
 youtube_dl/utils.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 65556d056..d39f313a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -259,15 +259,6 @@ def get_element_by_attribute(attribute, value, html):
     return unescapeHTML(res)
 
 
-def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s*["\']([^"\']+)["\']'):
-    attributes = re.findall(attributes_regex, attributes_str)
-    attributes_dict = {}
-    if attributes:
-        for (attribute_name, attribute_value) in attributes:
-            attributes_dict[attribute_name] = attribute_value
-    return attributes_dict
-
-
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 

From fd91257c4019a1956cc59eac1232f2c413b9747d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:08:36 +0600
Subject: [PATCH 1859/2145] [brightcove] Order imports alphabetically

---
 youtube_dl/extractor/brightcove.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ef34ae48f..f137ba8c6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -20,12 +20,12 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
-    unescapeHTML,
-    unsmuggle_url,
-    js_to_json,
     float_or_none,
+    js_to_json,
     int_or_none,
     parse_iso8601,
+    unescapeHTML,
+    unsmuggle_url,
 )
 
 

From e01b432ad38b36b1ba6cb1b6dccecec51f9fc1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:11:17 +0600
Subject: [PATCH 1860/2145] [brightcove:new] Fix test

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f137ba8c6..6b184157c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -366,7 +366,7 @@ class BrightcoveNewIE(InfoExtractor):
             'description': 'md5:eac376a4fe366edc70279bfb681aea16',
             'timestamp': 1441391203,
             'upload_date': '20150904',
-            'duration': 165768,
+            'duration': 165.768,
             'uploader_id': '929656772001',
         }
     }

From cb33d389ed452583638f73360599642b86617dfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:20:09 +0600
Subject: [PATCH 1861/2145] [brightcove:new] Add test with rtmp streams

---
 youtube_dl/extractor/brightcove.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 6b184157c..9ae724c2b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -356,7 +356,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 class BrightcoveNewIE(InfoExtractor):
     IE_NAME = 'brightcove:new'
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
         'info_dict': {
@@ -364,12 +364,30 @@ class BrightcoveNewIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Meet the man behind Popcorn Time',
             'description': 'md5:eac376a4fe366edc70279bfb681aea16',
+            'duration': 165.768,
             'timestamp': 1441391203,
             'upload_date': '20150904',
-            'duration': 165.768,
             'uploader_id': '929656772001',
+            'formats': 'mincount:22',
+        },
+    }, {
+        # with rtmp streams
+        'url': 'http://players.brightcove.net/4036320279001/5d112ed9-283f-485f-a7f9-33f42e8bc042_default/index.html?videoId=4279049078001',
+        'info_dict': {
+            'id': '4279049078001',
+            'ext': 'mp4',
+            'title': 'Titansgrave: Chapter 0',
+            'description': 'Titansgrave: Chapter 0',
+            'duration': 1242.058,
+            'timestamp': 1433556729,
+            'upload_date': '20150606',
+            'uploader_id': '4036320279001',
+            'formats': 'mincount:41',
+        },
+        'params': {
+            'skip_download': True,
         }
-    }
+    }]
 
     @staticmethod
     def _extract_urls(webpage):

From d8a1caf04f1733d7c94f213af25c587addcea35b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:22:12 +0600
Subject: [PATCH 1862/2145] [brightcove:new] Style

---
 youtube_dl/extractor/brightcove.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 9ae724c2b..14ee05f21 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -402,10 +402,11 @@ class BrightcoveNewIE(InfoExtractor):
         for _, url in re.findall(
                 r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url)
+
         # Look for embed_in_page embeds [2]
-        # According to examples from [3] it's unclear whether video id may be optional
-        # and what to do when it is
         for video_id, account_id, player_id, embed in re.findall(
+                # According to examples from [3] it's unclear whether video id
+                # may be optional and what to do when it is
                 r'''(?sx)
                     <video[^>]+
                         data-video-id=["\'](\d+)["\'][^>]*>.*?
@@ -417,6 +418,7 @@ class BrightcoveNewIE(InfoExtractor):
             entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
                 % (account_id, player_id, embed, video_id))
+
         return entries
 
     def _real_extract(self, url):

From a90189c3ad9bab0571121ecb70baea89ccac154e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 07:20:33 +0600
Subject: [PATCH 1863/2145] [instagram] Relax _VALID_URL (Closes #7497)

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 3d78f78c4..ddcaf76d1 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',

From 4479600d57d1a48de5a910764e2d18c30d7a4bd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 07:21:20 +0600
Subject: [PATCH 1864/2145] [instagram] Add test for #7497

---
 youtube_dl/extractor/instagram.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index ddcaf76d1..fce179000 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class InstagramIE(InfoExtractor):
     _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
         'info_dict': {
@@ -21,7 +21,10 @@ class InstagramIE(InfoExtractor):
             'title': 'Video by naomipq',
             'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
         }
-    }
+    }, {
+        'url': 'https://instagram.com/p/-Cmh1cukG2/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 5f1b2aea8051a34dd543ba0fd17e351a8b7bf858 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:02:07 +0800
Subject: [PATCH 1865/2145] [twitter:card] Support vine.co embeds (closes
 #7496)

---
 youtube_dl/extractor/twitter.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2bd5946ac..afbe8a651 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -52,6 +52,19 @@ class TwitterCardIE(InfoExtractor):
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
             },
+        },
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
+            'md5': 'ab2745d0b0ce53319a534fccaa986439',
+            'info_dict': {
+                'id': 'iBb2x00UVlv',
+                'ext': 'mp4',
+                'upload_date': '20151113',
+                'uploader_id': '1189339351084113920',
+                'uploader': '@ArsenalTerje',
+                'title': 'Vine by @ArsenalTerje',
+            },
+            'add_ie': ['Vine'],
         }
     ]
 
@@ -71,11 +84,11 @@ class TwitterCardIE(InfoExtractor):
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
-            youtube_url = self._html_search_regex(
-                r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
-                webpage, 'youtube iframe', default=None)
-            if youtube_url:
-                return self.url_result(youtube_url, 'Youtube')
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+                webpage, 'video iframe', default=None)
+            if iframe_url:
+                return self.url_result(iframe_url)
 
             config = self._parse_json(self._html_search_regex(
                 r'data-player-config="([^"]+)"', webpage, 'data player config'),

From 31752f76f76c6ca6f9e515b69a7488103575763e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:03:26 +0800
Subject: [PATCH 1866/2145] [twitter:card] Add add_ie for the external test

---
 youtube_dl/extractor/twitter.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index afbe8a651..055047340 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -52,6 +52,7 @@ class TwitterCardIE(InfoExtractor):
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
             },
+            'add_ie': ['Youtube'],
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',

From 9d584da7d0fd207d46e1592b25244cfcad4354ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:27:00 +0800
Subject: [PATCH 1867/2145] [xfileshare] Correct _VALID_URL

---
 youtube_dl/extractor/xfileshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 7610dc627..952515c98 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -19,7 +19,7 @@ class XFileShareIE(InfoExtractor):
     IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 

From 903d1369428f72397dbc698654dbad445ecaf2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 16:43:58 +0600
Subject: [PATCH 1868/2145] [lynda] Logout only when login info present (Closes
 #7500)

---
 youtube_dl/extractor/lynda.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 9a207b2cd..7d78a8805 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -83,6 +83,10 @@ class LyndaBaseIE(InfoExtractor):
             raise ExtractorError('Unable to log in')
 
     def _logout(self):
+        username, _ = self._get_login_info()
+        if username is None:
+            return
+
         self._download_webpage(
             'http://www.lynda.com/ajax/logout.aspx', None,
             'Logging out', 'Unable to log out', fatal=False)

From 0d85c3a7327e75173897d0e212254e496a46ea2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 16:44:24 +0600
Subject: [PATCH 1869/2145] [lynda] Style

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 7d78a8805..3d7e7e003 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -25,7 +25,7 @@ class LyndaBaseIE(InfoExtractor):
         self._login()
 
     def _login(self):
-        (username, password) = self._get_login_info()
+        username, password = self._get_login_info()
         if username is None:
             return
 

From dcdfeb33d2666ca07d2fb73ebf9284a77e714f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 22:32:54 +0600
Subject: [PATCH 1870/2145] [quickscope] Remove extractor

---
 youtube_dl/extractor/periscope.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 887c8020d..f4c1b622a 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -81,24 +81,3 @@ class PeriscopeIE(InfoExtractor):
             'thumbnails': thumbnails,
             'formats': formats,
         }
-
-
-class QuickscopeIE(InfoExtractor):
-    IE_DESC = 'Quick Scope'
-    _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://watchonperiscope.com/broadcast/56180087',
-        'only_matching': True,
-    }
-
-    def _real_extract(self, url):
-        broadcast_id = self._match_id(url)
-        request = compat_urllib_request.Request(
-            'https://watchonperiscope.com/api/accessChannel', compat_urllib_parse.urlencode({
-                'broadcast_id': broadcast_id,
-                'entry_ticket': '',
-                'from_push': 'false',
-                'uses_sessions': 'true',
-            }).encode('utf-8'))
-        return self.url_result(
-            self._download_json(request, broadcast_id)['share_url'], 'Periscope')

From 3b3e8ed332e44003f410d9f97967f0773e41ed1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 22:34:30 +0600
Subject: [PATCH 1871/2145] [quickscope] Remove extractor (2)

---
 youtube_dl/extractor/__init__.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 64ce3210b..eca4b00f5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -458,10 +458,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .periscope import (
-    PeriscopeIE,
-    QuickscopeIE,
-)
+from .periscope import PeriscopeIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE

From d781e293168cbdf93ee8d76e33b65387b13a2755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 23:08:13 +0600
Subject: [PATCH 1872/2145] [bbc] Allow selectionunavailable errors (Closes
 #7502)

---
 youtube_dl/extractor/bbc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a55a6dbc9..33b296eaf 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -27,7 +27,7 @@ class BBCCoUkIE(InfoExtractor):
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
         # with geolocation in some cases when it's even not geo restricted at all (e.g.
-        # http://www.bbc.co.uk/programmes/b06bp7lf)
+        # http://www.bbc.co.uk/programmes/b06bp7lf). Also may fail with selectionunavailable.
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
@@ -334,7 +334,7 @@ class BBCCoUkIE(InfoExtractor):
                 return self._download_media_selector_url(
                     mediaselector_url % programme_id, programme_id)
             except BBCCoUkIE.MediaSelectionError as e:
-                if e.id in ('notukerror', 'geolocation'):
+                if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
                     last_exception = e
                     continue
                 self._raise_extractor_error(e)
@@ -345,7 +345,7 @@ class BBCCoUkIE(InfoExtractor):
             media_selection = self._download_xml(
                 url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code in (403, 404):
                 media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise

From 0f72beb5153fbd4780a1c6b52e24acf5f0515874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Nov 2015 18:31:33 +0100
Subject: [PATCH 1873/2145] [periscope] Remove unused imports

---
 youtube_dl/extractor/periscope.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index f4c1b622a..459fa2c43 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,10 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
 from ..utils import parse_iso8601
 
 

From 0c59d02bdc86056957fcbebfec591ad53b7fcda3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 00:20:17 +0600
Subject: [PATCH 1874/2145] [periscope] Relax _VALID_URL (Closes #7503)

---
 youtube_dl/extractor/periscope.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 459fa2c43..63cc764bb 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
-    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
     # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -25,6 +25,9 @@ class PeriscopeIE(InfoExtractor):
     }, {
         'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
         'only_matching': True,
+    }, {
+        'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
+        'only_matching': True,
     }]
 
     def _call_api(self, method, value):

From dc0279532a218f5f43ca461aacd01058b8bc9e6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 02:21:24 +0600
Subject: [PATCH 1875/2145] [dumpert] Disable SSL (Closes #7504)

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 1f00386fe..8811be985 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -28,7 +28,7 @@ class DumpertIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        url = 'https://www.dumpert.nl/mediabase/' + video_id
+        url = 'http://www.dumpert.nl/mediabase/' + video_id
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)

From b2f77388305975570d57a4db3ced46aa3856f7a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 02:25:00 +0600
Subject: [PATCH 1876/2145] [dumpert] Use original protocol

---
 youtube_dl/extractor/dumpert.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 8811be985..f5a31058d 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
@@ -9,7 +10,7 @@ from ..utils import qualities
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
+    _VALID_URL = r'(?P<protocol>https?)://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -26,9 +27,11 @@ class DumpertIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        protocol = mobj.group('protocol')
 
-        url = 'http://www.dumpert.nl/mediabase/' + video_id
+        url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)

From 2ff7cbeaaabbc8d67caa224a5cef1fbd32761918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 08:30:13 +0600
Subject: [PATCH 1877/2145] [nowtv:list] Add extrator (Closes #7147)

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/nowtv.py    | 164 +++++++++++++++++++++----------
 2 files changed, 118 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eca4b00f5..59c82f65d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -420,7 +420,10 @@ from .nowness import (
     NownessPlaylistIE,
     NownessSeriesIE,
 )
-from .nowtv import NowTVIE
+from .nowtv import (
+    NowTVIE,
+    NowTVListIE,
+)
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index b0bdffc4e..67e34b294 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -13,8 +15,63 @@ from ..utils import (
 )
 
 
-class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+class NowTVBaseIE(InfoExtractor):
+    _VIDEO_FIELDS = (
+        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
+        'broadcastStartDate', 'seoUrl', 'duration', 'files',
+        'format.defaultImage169Format', 'format.defaultImage169Logo')
+
+    def _extract_video(self, info, display_id=None):
+        video_id = compat_str(info['id'])
+
+        files = info['files']
+        if not files:
+            if info.get('geoblocked', False):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
+
+        formats = []
+        for item in files['items']:
+            if determine_ext(item['path']) != 'f4v':
+                continue
+            app, play_path = remove_start(item['path'], '/').split('/', 1)
+            formats.append({
+                'url': 'rtmpe://fms.rtl.de',
+                'app': app,
+                'play_path': 'mp4:%s' % play_path,
+                'ext': 'flv',
+                'page_url': 'http://rtlnow.rtl.de',
+                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
+                'tbr': int_or_none(item.get('bitrate')),
+            })
+        self._sort_formats(formats)
+
+        title = info['title']
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
+        return {
+            'id': video_id,
+            'display_id': display_id or info.get('seoUrl'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class NowTVIE(NowTVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:list/[^/]+/)?(?P<id>[^/]+)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -23,7 +80,7 @@ class NowTVIE(InfoExtractor):
             'id': '203519',
             'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
             'ext': 'flv',
-            'title': 'Die neuen Bauern und eine Hochzeit',
+            'title': 'Inka Bause stellt die neuen Bauern vor',
             'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1432580700,
@@ -136,58 +193,65 @@ class NowTVIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        display_id_split = display_id.split('/')
-        if len(display_id) > 2:
-            display_id = '/'.join((display_id_split[0], display_id_split[-1]))
+        mobj = re.match(self._VALID_URL, url)
+        display_id = '%s/%s' % (mobj.group('show_id'), mobj.group('id'))
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
-            display_id)
+            'https://api.nowtv.de/v3/movies/%s?fields=%s'
+            % (display_id, ','.join(self._VIDEO_FIELDS)), display_id)
 
-        video_id = compat_str(info['id'])
+        return self._extract_video(info, display_id)
 
-        files = info['files']
-        if not files:
-            if info.get('geoblocked', False):
-                raise ExtractorError(
-                    'Video %s is not available from your location due to geo restriction' % video_id,
-                    expected=True)
-            if not info.get('free', True):
-                raise ExtractorError(
-                    'Video %s is not available for free' % video_id, expected=True)
 
-        formats = []
-        for item in files['items']:
-            if determine_ext(item['path']) != 'f4v':
-                continue
-            app, play_path = remove_start(item['path'], '/').split('/', 1)
-            formats.append({
-                'url': 'rtmpe://fms.rtl.de',
-                'app': app,
-                'play_path': 'mp4:%s' % play_path,
-                'ext': 'flv',
-                'page_url': 'http://rtlnow.rtl.de',
-                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
-                'tbr': int_or_none(item.get('bitrate')),
-            })
-        self._sort_formats(formats)
+class NowTVListIE(NowTVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/list/(?P<id>[^?/#&]+)$'
 
-        title = info['title']
-        description = info.get('articleLong') or info.get('articleShort')
-        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
-        duration = parse_duration(info.get('duration'))
+    _SHOW_FIELDS = ('title', )
+    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
 
-        f = info.get('format', {})
-        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+    _TESTS = [{
+        'url': 'http://www.nowtv.at/rtl/stern-tv/list/aktuell',
+        'info_dict': {
+            'id': '17006',
+            'title': 'stern TV - Aktuell',
+        },
+        'playlist_count': 1,
+    }, {
+        'url': 'http://www.nowtv.at/rtl/das-supertalent/list/free-staffel-8',
+        'info_dict': {
+            'id': '20716',
+            'title': 'Das Supertalent - FREE Staffel 8',
+        },
+        'playlist_count': 14,
+    }]
 
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-        }
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        show_id = mobj.group('show_id')
+        season_id = mobj.group('id')
+
+        fields = []
+        fields.extend(self._SHOW_FIELDS)
+        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
+        fields.extend(
+            'formatTabs.formatTabPages.container.movies.%s' % field
+            for field in self._VIDEO_FIELDS)
+
+        list_info = self._download_json(
+            'https://api.nowtv.de/v3/formats/seo?fields=%s&name=%s.php'
+            % (','.join(fields), show_id),
+            season_id)
+
+        season = next(
+            season for season in list_info['formatTabs']['items']
+            if season.get('seoheadline') == season_id)
+
+        title = '%s - %s' % (list_info['title'], season['headline'])
+
+        entries = []
+        for container in season['formatTabPages']['items']:
+            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
+                entries.append(self._extract_video(info))
+
+        return self.playlist_result(
+            entries, compat_str(season.get('id') or season_id), title)

From 828b2a5cd971ae04469aeb7b11dfeaab7962c4d9 Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Sun, 15 Nov 2015 09:40:32 +0200
Subject: [PATCH 1878/2145] Removing an unnecessary import

---
 youtube_dl/update.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 04bf0939e..074eb64a7 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -9,10 +9,7 @@ import subprocess
 import sys
 from zipimport import zipimporter
 
-from .compat import (
-    compat_str,
-    compat_urllib_request,
-)
+from .compat import compat_str
 
 from .version import __version__
 

From ad1f4e7902464532ba11f3a09ac6ff2312b01661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 23:43:23 +0600
Subject: [PATCH 1879/2145] [theplatform] Handle explicitly specified SMIL
 (#7385)

---
 youtube_dl/extractor/theplatform.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 25edc3100..114c70b7c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -193,6 +193,15 @@ class ThePlatformIE(ThePlatformBaseIE):
 
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
+        # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
+        elif '/guid/' in url:
+            webpage = self._download_webpage(url, video_id)
+            smil_url = self._search_regex(
+                r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
+                webpage, 'smil url', group='url')
+            path = self._search_regex(
+                r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
+            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL'
         elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')

From 9a4acbfaf58f7334bdd8a3b66dadf331f1ce1d7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 00:28:04 +0600
Subject: [PATCH 1880/2145] [theplatform] Add test for #7385

---
 youtube_dl/extractor/theplatform.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 114c70b7c..43315e75d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -139,6 +139,11 @@ class ThePlatformIE(ThePlatformBaseIE):
             'upload_date': '20150701',
             'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
         },
+    }, {
+        # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
+        # geo-restricted (US), HLS encrypted with AES-128
+        'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
+        'only_matching': True,
     }]
 
     @staticmethod

From bd1512d19649c280197729814766d590ea6c023b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 15 Nov 2015 22:16:08 +0100
Subject: [PATCH 1881/2145] release 2015.11.15

---
 docs/supportedsites.md | 5 +++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5016ba4bc..2e5283747 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,7 +67,8 @@
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
- - **Brightcove**
+ - **brightcove:legacy**
+ - **brightcove:new**
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
@@ -366,6 +367,7 @@
  - **nowness:playlist**
  - **nowness:series**
  - **NowTV**
+ - **NowTVList**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
@@ -425,7 +427,6 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
- - **Quickscope**: Quick Scope
  - **QuickVid**
  - **R7**
  - **radio.de**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6585d60d5..6f601cbb1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.13'
+__version__ = '2015.11.15'

From 76adc820682b78b67ffc9cb1544807f7e021e73b Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 16 Nov 2015 11:39:18 +0800
Subject: [PATCH 1882/2145] [neteasemusic] Fixes #7301

---
 youtube_dl/extractor/neteasemusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index a8e0a64ed..bb3362069 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -40,7 +40,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
             if not details:
                 continue
             formats.append({
-                'url': 'http://m1.music.126.net/%s/%s.%s' %
+                'url': 'http://m5.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
                 'ext': details.get('extension'),

From 741dd8ea65b276997f3eadae43b2879e9a229a80 Mon Sep 17 00:00:00 2001
From: Rastislav Barlik <barlik@zoho.com>
Date: Mon, 16 Nov 2015 14:15:25 +0000
Subject: [PATCH 1883/2145] Clarify that automatic subtitles are generated.

It wasn't clear what automatic word mean.
---
 README.md               | 4 ++--
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/options.py   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 38db97c59..b286651cd 100644
--- a/README.md
+++ b/README.md
@@ -329,8 +329,8 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Subtitle Options:
     --write-sub                      Write subtitle file
-    --write-auto-sub                 Write automatic subtitle file (YouTube
-                                     only)
+    --write-auto-sub                 Write automatically generated subtitle file
+                                     (YouTube only)
     --all-subs                       Download all the available subtitles of the
                                      video
     --list-subs                      List all available subtitles for the video
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..422f3ffca 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -156,7 +156,7 @@ class YoutubeDL(object):
     writethumbnail:    Write the thumbnail image to a file
     write_all_thumbnails:  Write all thumbnail formats to files
     writesubtitles:    Write the video subtitles to a file
-    writeautomaticsub: Write the automatic subtitles to a file
+    writeautomaticsub: Write the automatically generated subtitles to a file
     allsubtitles:      Downloads all the subtitles of the video
                        (requires writesubtitles or writeautomaticsub)
     listsubtitles:     Lists all available subtitles for the video
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3dd6d290b..079fe7e8a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -363,7 +363,7 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--write-auto-sub', '--write-automatic-sub',
         action='store_true', dest='writeautomaticsub', default=False,
-        help='Write automatic subtitle file (YouTube only)')
+        help='Write automatically generated subtitle file (YouTube only)')
     subtitles.add_option(
         '--all-subs',
         action='store_true', dest='allsubtitles', default=False,

From 7aefc49c4013efb5056b2c1237e22c52cb5d3c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 20:20:16 +0600
Subject: [PATCH 1884/2145] [utils] Skip invalid/non HTML entities (Closes
 #7518)

---
 test/test_utils.py  | 4 ++--
 youtube_dl/utils.py | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 01829f71e..ea1ff0547 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -210,8 +210,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unescapeHTML('%20;'), '%20;')
         self.assertEqual(unescapeHTML('&#x2F;'), '/')
         self.assertEqual(unescapeHTML('&#47;'), '/')
-        self.assertEqual(
-            unescapeHTML('&eacute;'), 'é')
+        self.assertEqual(unescapeHTML('&eacute;'), 'é')
+        self.assertEqual(unescapeHTML('&#2013266066;'), '&#2013266066;')
 
     def test_daterange(self):
         _20century = DateRange("19000101", "20000101")
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d39f313a4..b7013a6aa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -396,7 +396,11 @@ def _htmlentity_transform(entity):
             numstr = '0%s' % numstr
         else:
             base = 10
-        return compat_chr(int(numstr, base))
+        # See https://github.com/rg3/youtube-dl/issues/7518
+        try:
+            return compat_chr(int(numstr, base))
+        except ValueError:
+            pass
 
     # Unknown entity in name, return its literal representation
     return ('&%s;' % entity)

From 7a3f0c00ad138eef396ae8fd1583fe29b4c4c684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 20:24:09 +0600
Subject: [PATCH 1885/2145] [utils] Style

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b7013a6aa..d00b14b86 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -403,7 +403,7 @@ def _htmlentity_transform(entity):
             pass
 
     # Unknown entity in name, return its literal representation
-    return ('&%s;' % entity)
+    return '&%s;' % entity
 
 
 def unescapeHTML(s):

From 9b464929fe62d405daccf866cdfdfa6257208839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Nov 2015 21:11:42 +0600
Subject: [PATCH 1886/2145] [rtve.es:alacarta] Fix extraction

---
 youtube_dl/extractor/rtve.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 5b97d33ca..81fcfedc7 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -107,15 +107,9 @@ class RTVEALaCartaIE(InfoExtractor):
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
-            auth_url = video_url.replace(
+            video_url = video_url.replace(
                 'resources/', 'auth/resources/'
             ).replace('.net.rtve', '.multimedia.cdn.rtve')
-            video_path = self._download_webpage(
-                auth_url, video_id, 'Getting video url')
-            # Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
-            # the right Content-Length header and the mp4 format
-            video_url = compat_urlparse.urljoin(
-                'http://mvod1.akcdn.rtve.es/', video_path)
 
         subtitles = None
         if info.get('sbtFile') is not None:

From e156e70281332b14c3aa47a71fd4a7a16be1a225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 16:23:29 +0100
Subject: [PATCH 1887/2145] [rtve] Remove unused import

---
 youtube_dl/extractor/rtve.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 81fcfedc7..0fe6356db 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,7 +6,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request, compat_urlparse
+from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     float_or_none,

From 4cd759f73d734882b6d74e764abb068e64a99635 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 17:52:29 +0100
Subject: [PATCH 1888/2145] [dplay] Add extractor (closes #7515)

Since I haven't figured out how to download the hds stream, we use the hls one instead.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dplay.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/dplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 59c82f65d..26e5745d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -132,6 +132,7 @@ from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
+from .dplay import DPlayIE
 from .dramafever import (
     DramaFeverIE,
     DramaFeverSeriesIE,
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
new file mode 100644
index 000000000..93768c294
--- /dev/null
+++ b/youtube_dl/extractor/dplay.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+import time
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class DPlayIE(InfoExtractor):
+    _VALID_URL = r'http://www\.dplay\.se/[^/]+/(?P<id>[^/?#]+)'
+
+    _TEST = {
+        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+        'info_dict': {
+            'id': '3172',
+            'ext': 'mp4',
+            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+            'title': 'Svensken lär sig njuta av livet',
+            'duration': 2650,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r'data-video-id="(\d+)"', webpage, 'video id')
+
+        info = self._download_json(
+            'http://www.dplay.se/api/v2/ajax/videos?video_id=' + video_id,
+            video_id)['data'][0]
+
+        self._set_cookie(
+            'secure.dplay.se', 'dsc-geo',
+            '{"countryCode":"NL","expiry":%d}' % ((time.time() + 20 * 60) * 1000))
+        # TODO: consider adding support for 'stream_type=hds', it seems to
+        # require setting some cookies
+        manifest_url = self._download_json(
+            'https://secure.dplay.se/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % video_id,
+            video_id, 'Getting manifest url for hls stream')['hls']
+        formats = self._extract_m3u8_formats(
+            manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': info['title'],
+            'formats': formats,
+            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+        }

From 609af1ae1ce578b0fdd1dd7ec6e56f1272e107f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 17:57:40 +0100
Subject: [PATCH 1889/2145] [dplay] Add 'encoding: utf-8' line

---
 youtube_dl/extractor/dplay.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 93768c294..6cda56a7f 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import unicode_literals
 
 import time

From 312a3f389b4c788556cfcceec1556580c9a9520f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Nov 2015 00:46:41 +0600
Subject: [PATCH 1890/2145] [pbs] Extend _VALID_URL

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 8fb9b1849..33d5c1cf7 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -22,7 +22,7 @@ class PBSIE(InfoExtractor):
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
-           video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
+           (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
     '''
 
@@ -170,6 +170,10 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
+            'only_matching': True,
         }
     ]
     _ERRORS = {

From 63b4295d20a5b98a9fca4dc3ce132b26408110d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Nov 2015 18:28:05 +0100
Subject: [PATCH 1891/2145] [youtube:playlist] fix title extraction (fixes
 #7544 and #7545)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 687e0b4db..364ca102a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1615,7 +1615,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtract
                 self.report_warning('Youtube gives an alert message: ' + match)
 
         playlist_title = self._html_search_regex(
-            r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
+            r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
             page, 'title')
 
         return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)

From 82beaabb41f015b4aaa1b7460a881a9653e398e9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Nov 2015 19:23:04 +0100
Subject: [PATCH 1892/2145] release 2015.11.18

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2e5283747..f56a6986b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -129,6 +129,7 @@
  - **Discovery**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
+ - **DPlay**
  - **dramafever**
  - **dramafever:series**
  - **DRBonanza**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6f601cbb1..b394cb880 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.15'
+__version__ = '2015.11.18'

From a9c09a7c62df3024b0774de7204e9262bb2d7d4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 20:25:28 +0600
Subject: [PATCH 1893/2145] [pbs] Update API URL (Closes #7565)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 33d5c1cf7..b787e2a73 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -263,7 +263,7 @@ class PBSIE(InfoExtractor):
             return self.playlist_result(entries, display_id)
 
         info = self._download_json(
-            'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+            'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
             display_id)
 
         formats = []

From c39fd7b1ca3e81901aa28d36523fe698295e58d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 19 Nov 2015 22:22:57 +0800
Subject: [PATCH 1894/2145] [UDNEmbed] Fix generic UDN pages

Closes #7547
---
 youtube_dl/extractor/generic.py | 2 +-
 youtube_dl/extractor/udn.py     | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 51516a38a..b483eba65 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1739,7 +1739,7 @@ class GenericIE(InfoExtractor):
 
         # Look for UDN embeds
         mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
+            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index 2151f8338..ee35b7227 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -12,7 +12,8 @@ from ..compat import compat_urlparse
 
 class UDNEmbedIE(InfoExtractor):
     IE_DESC = '聯合影音'
-    _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
+    _PROTOCOL_RELATIVE_VALID_URL = r'//video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
+    _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',
         'md5': 'de06b4c90b042c128395a88f0384817e',

From e8110b81254075462fd39590dd05d7cd0214cf22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Nov 2015 15:35:13 +0100
Subject: [PATCH 1895/2145] release 2015.11.19

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b394cb880..b3b1b90ba 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.18'
+__version__ = '2015.11.19'

From 2c94198eb62a9ce46a09badfdb232f8b5243a893 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 21:29:32 +0600
Subject: [PATCH 1896/2145] [vimeo] Improve playlists extraction

---
 youtube_dl/extractor/vimeo.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b72341a2b..cb8c0d484 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -486,8 +486,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
             password_request, list_id,
             'Verifying the password', 'Wrong password')
 
-    def _extract_videos(self, list_id, base_url):
-        video_ids = []
+    def _title_and_entries(self, list_id, base_url):
         for pagenum in itertools.count(1):
             page_url = self._page_url(base_url, pagenum)
             webpage = self._download_webpage(
@@ -496,18 +495,18 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
 
             if pagenum == 1:
                 webpage = self._login_list_password(page_url, list_id, webpage)
+                yield self._extract_list_title(webpage)
+
+            for video_id in re.findall(r'id="clip_(\d+?)"', webpage):
+                yield self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
 
-            video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
 
-        entries = [self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
-                   for video_id in video_ids]
-        return {'_type': 'playlist',
-                'id': list_id,
-                'title': self._extract_list_title(webpage),
-                'entries': entries,
-                }
+    def _extract_videos(self, list_id, base_url):
+        title_and_entries = self._title_and_entries(list_id, base_url)
+        list_title = next(title_and_entries)
+        return self.playlist_result(title_and_entries, list_id, list_title)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From fdb20a27a335dddeebd2e6535a1b9c12c08aa42f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:30:58 +0600
Subject: [PATCH 1897/2145] [vimeo:group] Improve _VALID_URL (Closes #7552)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cb8c0d484..836a7312b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -567,7 +567,7 @@ class VimeoAlbumIE(VimeoChannelIE):
 
 class VimeoGroupsIE(VimeoAlbumIE):
     IE_NAME = 'vimeo:group'
-    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)'
+    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)(?:/(?!videos?/\d+)|$)'
     _TESTS = [{
         'url': 'https://vimeo.com/groups/rolexawards',
         'info_dict': {

From 6b7ceee1b9124521d2f583fb0d0717999747fdcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:31:16 +0600
Subject: [PATCH 1898/2145] [vimeo] Add test for #7552

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 836a7312b..057c72f39 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -189,6 +189,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'note': 'Video not completely processed, "failed" seed status',
             'only_matching': True,
         },
+        {
+            'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
+            'only_matching': True,
+        },
     ]
 
     @staticmethod

From 371c3b796cb4168dbb2cb1ad48e3cf12745a8601 Mon Sep 17 00:00:00 2001
From: hedii <hedi.chaibs@gmail.com>
Date: Tue, 17 Nov 2015 16:41:59 +0100
Subject: [PATCH 1899/2145] [YoutubeDL] Add playlist finished downloading
 message (Closes #7517)

Conflicts:
	youtube_dl/YoutubeDL.py
---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..9d626049a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -833,6 +833,7 @@ class YoutubeDL(object):
                                                       extra_info=extra)
                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
+            self.to_screen('[download] Finished downloading playlist: %s' % playlist)
             return ie_result
         elif result_type == 'compat_list':
             self.report_warning(

From 342609a1b452c8b8a02e86c3de2ec2a55b6e809b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:55:06 +0600
Subject: [PATCH 1900/2145] [bloomberg] Reax _VALID_URL (Closes #7546)

---
 youtube_dl/extractor/bloomberg.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 0dca29b71..11ace91dd 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/news/videos/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://www\.bloomberg\.com/news/[^/]+/[^/]+/(?P<id>[^/?#]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
         # The md5 checksum changes
         'info_dict': {
@@ -17,7 +17,10 @@ class BloombergIE(InfoExtractor):
             'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
             'description': 'md5:a8ba0302912d03d246979735c17d2761',
         },
-    }
+    }, {
+        'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         name = self._match_id(url)

From ee5cd8418e7d6503bfa33ef44e2eb863a55a8c8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:58:29 +0600
Subject: [PATCH 1901/2145] [theplatform] Handle protocolless feed URLs (Closes
 #7532)

---
 youtube_dl/extractor/theplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 43315e75d..d52f9ae4b 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -187,7 +187,8 @@ class ThePlatformIE(ThePlatformBaseIE):
             # Seems there's no pattern for the interested script filename, so
             # I try one by one
             for script in reversed(scripts):
-                feed_script = self._download_webpage(script, video_id, 'Downloading feed script')
+                feed_script = self._download_webpage(
+                    self._proto_relative_url(script, 'http:'), video_id, 'Downloading feed script')
                 feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
                 if feed_id is not None:
                     break

From 325bb615a79928541aabfbab366ef813b4f72309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:58:43 +0600
Subject: [PATCH 1902/2145] [theplatform] Style

---
 youtube_dl/extractor/theplatform.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index d52f9ae4b..1555aa77c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -188,8 +188,11 @@ class ThePlatformIE(ThePlatformBaseIE):
             # I try one by one
             for script in reversed(scripts):
                 feed_script = self._download_webpage(
-                    self._proto_relative_url(script, 'http:'), video_id, 'Downloading feed script')
-                feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
+                    self._proto_relative_url(script, 'http:'),
+                    video_id, 'Downloading feed script')
+                feed_id = self._search_regex(
+                    r'defaultFeedId\s*:\s*"([^"]+)"', feed_script,
+                    'default feed id', default=None)
                 if feed_id is not None:
                     break
             if feed_id is None:

From 67446fd49b447531720d62aa339fc364a1699e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 04:07:39 +0600
Subject: [PATCH 1903/2145] [instagram] Improve _VALID_URL (Closes #7568)

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index fce179000..c158f2064 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/p/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',

From 17cc1534359aad2c180cfd8a4f66d9d0eccb9e01 Mon Sep 17 00:00:00 2001
From: ashutosh-mishra <ashutosh.mishra@oneconvergence.com>
Date: Fri, 20 Nov 2015 22:51:46 +0530
Subject: [PATCH 1904/2145] Typo fix, found while going through the code.

---
 youtube_dl/YoutubeDL.py         | 2 +-
 youtube_dl/downloader/common.py | 2 +-
 youtube_dl/downloader/rtmp.py   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2e824117a..fba99af8d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -938,7 +938,7 @@ class YoutubeDL(object):
                     filter_parts.append(string)
 
         def _remove_unused_ops(tokens):
-            # Remove operators that we don't use and join them with the sourrounding strings
+            # Remove operators that we don't use and join them with the surrounding strings
             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
             ALLOWED_OPS = ('/', '+', ',', '(', ')')
             last_string, last_start, last_end, last_line = None, None, None, None
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 29a4500d3..b8bf8daf8 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -42,7 +42,7 @@ class FileDownloader(object):
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
-                        (experimenatal)
+                        (experimental)
     external_downloader_args:  A list of additional command-line arguments for the
                         external downloader.
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index f1d219ba9..14d56db47 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -117,7 +117,7 @@ class RtmpFD(FileDownloader):
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
-        # the connection was interrumpted and resuming appears to be
+        # the connection was interrupted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
         basic_args = [
             'rtmpdump', '--verbose', '-r', url,

From 01b06aedcf9692e03a80efe7e9857560a2a6eb45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:34:02 +0600
Subject: [PATCH 1905/2145] [kaltura] Add support for referrer protected videos
 (#7409)

---
 youtube_dl/extractor/kaltura.py | 47 ++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 0dcd6cd05..39e038659 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -2,12 +2,17 @@
 from __future__ import unicode_literals
 
 import re
+import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
+    unsmuggle_url,
 )
 
 
@@ -121,24 +126,40 @@ class KalturaIE(InfoExtractor):
             video_id, actions, note='Downloading video info JSON')
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
         partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
         entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
 
         info, source_data = self._get_video_info(entry_id, partner_id)
 
-        formats = [{
-            'format_id': '%(fileExt)s-%(bitrate)s' % f,
-            'ext': f['fileExt'],
-            'tbr': f['bitrate'],
-            'fps': f.get('frameRate'),
-            'filesize_approx': int_or_none(f.get('size'), invscale=1024),
-            'container': f.get('containerFormat'),
-            'vcodec': f.get('videoCodecId'),
-            'height': f.get('height'),
-            'width': f.get('width'),
-            'url': '%s/flavorId/%s' % (info['dataUrl'], f['id']),
-        } for f in source_data['flavorAssets']]
+        source_url = smuggled_data.get('source_url')
+        if source_url:
+            referrer = base64.b64encode(
+                '://'.join(compat_urlparse.urlparse(source_url)[:2])
+                .encode('utf-8')).decode('utf-8')
+        else:
+            referrer = None
+
+        formats = []
+        for f in source_data['flavorAssets']:
+            video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
+            if referrer:
+                video_url += '?referrer=%s' % referrer
+            formats.append({
+                'format_id': '%(fileExt)s-%(bitrate)s' % f,
+                'ext': f['fileExt'],
+                'tbr': f['bitrate'],
+                'fps': f.get('frameRate'),
+                'filesize_approx': int_or_none(f.get('size'), invscale=1024),
+                'container': f.get('containerFormat'),
+                'vcodec': f.get('videoCodecId'),
+                'height': f.get('height'),
+                'width': f.get('width'),
+                'url': video_url,
+            })
+        self._check_formats(formats, entry_id)
         self._sort_formats(formats)
 
         return {

From 5b5fae5f2004e5a1be36551bf477f74161d4f9ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:35:58 +0600
Subject: [PATCH 1906/2145] [generic] Use referrer from source kaltura embed
 URLs (#7409)

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b483eba65..c7cee5487 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1694,7 +1694,9 @@ class GenericIE(InfoExtractor):
         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
-            return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
+            return self.url_result(smuggle_url(
+                'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
+                {'source_url': url}), 'Kaltura')
 
         # Look for Eagle.Platform embeds
         mobj = re.search(

From d80a39cec89e8ddfc3090c3d1b10d3fe105791e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:38:08 +0600
Subject: [PATCH 1907/2145] [kaltura] Improve

---
 youtube_dl/extractor/kaltura.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 39e038659..1eaa09fea 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -149,14 +149,14 @@ class KalturaIE(InfoExtractor):
                 video_url += '?referrer=%s' % referrer
             formats.append({
                 'format_id': '%(fileExt)s-%(bitrate)s' % f,
-                'ext': f['fileExt'],
-                'tbr': f['bitrate'],
-                'fps': f.get('frameRate'),
+                'ext': f.get('fileExt'),
+                'tbr': int_or_none(f['bitrate']),
+                'fps': int_or_none(f.get('frameRate')),
                 'filesize_approx': int_or_none(f.get('size'), invscale=1024),
                 'container': f.get('containerFormat'),
                 'vcodec': f.get('videoCodecId'),
-                'height': f.get('height'),
-                'width': f.get('width'),
+                'height': int_or_none(f.get('height')),
+                'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
         self._check_formats(formats, entry_id)

From bdceea7afde4fe02fff90f84ef1f2fb6ebbac9d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:39:29 +0600
Subject: [PATCH 1908/2145] [kaltura] Clean description

---
 youtube_dl/extractor/kaltura.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 1eaa09fea..583b1a5ad 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    clean_html,
     ExtractorError,
     int_or_none,
     unsmuggle_url,
@@ -166,7 +167,7 @@ class KalturaIE(InfoExtractor):
             'id': entry_id,
             'title': info['name'],
             'formats': formats,
-            'description': info.get('description'),
+            'description': clean_html(info.get('description')),
             'thumbnail': info.get('thumbnailUrl'),
             'duration': info.get('duration'),
             'timestamp': info.get('createdAt'),

From 6da620de58bd1a4221bcdae5f4f2098b247038bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:40:28 +0600
Subject: [PATCH 1909/2145] [kaltura] Add test for referrer protected video
 (#7409)

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7cee5487..e512c1e14 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -823,6 +823,19 @@ class GenericIE(InfoExtractor):
                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
             },
         },
+        # Kaltura embed protected with referrer
+        {
+            'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
+            'info_dict': {
+                'id': '1_g4fbemnq',
+                'ext': 'mp4',
+                'title': 'Violetta - Achter De Schermen - Ruggero',
+                'description': 'Achter de schermen met Ruggero',
+                'timestamp': 1435133761,
+                'upload_date': '20150624',
+                'uploader_id': 'echojecka',
+            },
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',

From 71bd93b89c667b7ca852b3a536dd771da7b67f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:08:34 +0600
Subject: [PATCH 1910/2145] [pluralsight] Do not rely on argument order in
 query (Closes #7583)

---
 youtube_dl/extractor/pluralsight.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index fd32836cc..6cef7c829 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import json
 
 from .common import InfoExtractor
@@ -19,11 +18,11 @@ from ..utils import (
 
 class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
-    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
+    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
     _LOGIN_URL = 'https://www.pluralsight.com/id/'
     _NETRC_MACHINE = 'pluralsight'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
         'md5': '4d458cf5cf4c593788672419a8dd4cf8',
         'info_dict': {
@@ -33,7 +32,10 @@ class PluralsightIE(InfoExtractor):
             'duration': 338,
         },
         'skip': 'Requires pluralsight account credentials',
-    }
+    }, {
+        'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
+        'only_matching': True,
+    }]
 
     def _real_initialize(self):
         self._login()
@@ -74,11 +76,15 @@ class PluralsightIE(InfoExtractor):
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        author = mobj.group('author')
-        name = mobj.group('name')
-        clip_id = mobj.group('clip')
-        course = mobj.group('course')
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+
+        author = qs.get('author', [None])[0]
+        name = qs.get('name', [None])[0]
+        clip_id = qs.get('clip', [None])[0]
+        course = qs.get('course', [None])[0]
+
+        if any(not f for f in (author, name, clip_id, course,)):
+            raise ExtractorError('Invalid URL', expected=True)
 
         display_id = '%s-%s' % (name, clip_id)
 

From 651acffbe57a016ae3b6b2c842aa691b1feb2d5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:21:33 +0600
Subject: [PATCH 1911/2145] [pluralsight] Update ViewClip URL

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 6cef7c829..caba8fb79 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -138,7 +138,7 @@ class PluralsightIE(InfoExtractor):
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
                 request = compat_urllib_request.Request(
-                    'http://www.pluralsight.com/training/Player/ViewClip',
+                    'http://app.pluralsight.com/training/Player/ViewClip',
                     json.dumps(clip_post).encode('utf-8'))
                 request.add_header('Content-Type', 'application/json;charset=utf-8')
                 format_id = '%s-%s' % (ext, quality)

From c23e266427ef7fdcff3dc02e0d9978c58addea6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:25:52 +0600
Subject: [PATCH 1912/2145] [pluralsight] Do not require pluralsight account

Looks like some courses are available without pluralsight account
---
 youtube_dl/extractor/pluralsight.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index caba8fb79..c6fd41d5a 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -35,6 +35,10 @@ class PluralsightIE(InfoExtractor):
     }, {
         'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
         'only_matching': True,
+    }, {
+        # available without pluralsight account
+        'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
@@ -43,7 +47,7 @@ class PluralsightIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            self.raise_login_required('Pluralsight account is required')
+            return
 
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
@@ -172,7 +176,7 @@ class PluralsightIE(InfoExtractor):
 class PluralsightCourseIE(InfoExtractor):
     IE_NAME = 'pluralsight:course'
     _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         # Free course from Pluralsight Starter Subscription for Microsoft TechNet
         # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
         'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
@@ -182,7 +186,11 @@ class PluralsightCourseIE(InfoExtractor):
             'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
         },
         'playlist_count': 31,
-    }
+    }, {
+        # available without pluralsight account
+        'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         course_id = self._match_id(url)

From a5cd0eb8a4f2661cfe4863e958172d8f8f42d0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:32:48 +0600
Subject: [PATCH 1913/2145] [pluralsight:course] Improve _VALID_URL

---
 youtube_dl/extractor/pluralsight.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index c6fd41d5a..93244f41d 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -175,7 +175,7 @@ class PluralsightIE(InfoExtractor):
 
 class PluralsightCourseIE(InfoExtractor):
     IE_NAME = 'pluralsight:course'
-    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
     _TESTS = [{
         # Free course from Pluralsight Starter Subscription for Microsoft TechNet
         # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
@@ -190,6 +190,9 @@ class PluralsightCourseIE(InfoExtractor):
         # available without pluralsight account
         'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
         'only_matching': True,
+    }, {
+        'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 6cc37c69e25f886c466d4b89f0734639e20e279b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 14:12:34 +0800
Subject: [PATCH 1914/2145] [generic] Unescape URLs from JWPlayer (#7582)

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e512c1e14..6cffde20d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1874,6 +1874,7 @@ class GenericIE(InfoExtractor):
 
         entries = []
         for video_url in found:
+            video_url = video_url.replace('\\/', '/')
             video_url = compat_urlparse.urljoin(url, video_url)
             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
 

From 28602e747c13a7979aedd517e491bada3856cb12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 16:08:54 +0800
Subject: [PATCH 1915/2145] [generic] Refactor

---
 youtube_dl/extractor/generic.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6cffde20d..1991a8684 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1886,25 +1886,22 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
+            entry_info_dict = {
+                'id': video_id,
+                'uploader': video_uploader,
+                'title': video_title,
+                'age_limit': age_limit,
+            }
+
             ext = determine_ext(video_url)
             if ext == 'smil':
-                entries.append({
-                    'id': video_id,
-                    'formats': self._extract_smil_formats(video_url, video_id),
-                    'uploader': video_uploader,
-                    'title': video_title,
-                    'age_limit': age_limit,
-                })
+                entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
             elif ext == 'xspf':
                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             else:
-                entries.append({
-                    'id': video_id,
-                    'url': video_url,
-                    'uploader': video_uploader,
-                    'title': video_title,
-                    'age_limit': age_limit,
-                })
+                entry_info_dict['url'] = video_url
+
+            entries.append(entry_info_dict)
 
         if len(entries) == 1:
             return entries[0]

From 750b9ff0324b04f252b05cfadd802e8b1f2cc19d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 16:43:01 +0800
Subject: [PATCH 1916/2145] [generic] Extract M3U8 formats (closes #7582)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1991a8684..2b934148d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1058,6 +1058,20 @@ class GenericIE(InfoExtractor):
                 'description': 'Tabletop: Dread, Last Thoughts',
                 'duration': 51690,
             },
+        },
+        # JWPlayer with M3U8
+        {
+            'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
+            'info_dict': {
+                'id': 'playlist',
+                'ext': 'mp4',
+                'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
+                'uploader': 'ren.tv',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
         }
     ]
 
@@ -1898,6 +1912,8 @@ class GenericIE(InfoExtractor):
                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
             elif ext == 'xspf':
                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
+            elif ext == 'm3u8':
+                entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
             else:
                 entry_info_dict['url'] = video_url
 

From f52183a8780a7b9e58ffb4510e234a7392ad8a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 17:39:24 +0600
Subject: [PATCH 1917/2145] [rutube:embed] Extend _VALID_URL (Closes #7588)

---
 youtube_dl/extractor/rutube.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index d94dc7399..e824129b4 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -74,9 +74,9 @@ class RutubeIE(InfoExtractor):
 class RutubeEmbedIE(InfoExtractor):
     IE_NAME = 'rutube:embed'
     IE_DESC = 'Rutube embedded videos'
-    _VALID_URL = 'https?://rutube\.ru/video/embed/(?P<id>[0-9]+)'
+    _VALID_URL = 'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
         'info_dict': {
             'id': 'a10e53b86e8f349080f718582ce4c661',
@@ -90,7 +90,10 @@ class RutubeEmbedIE(InfoExtractor):
         'params': {
             'skip_download': 'Requires ffmpeg',
         },
-    }
+    }, {
+        'url': 'http://rutube.ru/play/embed/8083783',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         embed_id = self._match_id(url)

From 019839faaa28d6233097a669d063a3a42f9fbbc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 18:01:39 +0600
Subject: [PATCH 1918/2145] [extractor/common] Use baseURL from f4m manifest
 for recursive manifest extraction

---
 youtube_dl/extractor/common.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5e263f8b5..71bdcad5a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -891,6 +891,11 @@ class InfoExtractor(object):
         if not media_nodes:
             manifest_version = '2.0'
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
+        base_url = xpath_text(
+            manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
+            'base URL', default=None)
+        if base_url:
+            base_url = base_url.strip()
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
                 media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
@@ -898,7 +903,7 @@ class InfoExtractor(object):
                     continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
-                    else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+                    else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
                 # If media_url is itself a f4m manifest do the recursive extraction
                 # since bitrates in parent manifest (this one) and media_url manifest
                 # may differ leading to inability to resolve the format by requested

From 413719689996b2642f2484dd572d551376f0d104 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 18:02:52 +0600
Subject: [PATCH 1919/2145] [rutube] Extract all formats

---
 youtube_dl/extractor/rutube.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index e824129b4..0db8c410d 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     unified_strdate,
 )
 
@@ -51,10 +51,26 @@ class RutubeIE(InfoExtractor):
             'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
             video_id, 'Downloading options JSON')
 
-        m3u8_url = options['video_balancer'].get('m3u8')
-        if m3u8_url is None:
-            raise ExtractorError('Couldn\'t find m3u8 manifest url')
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        formats = []
+        for format_id, format_url in options['video_balancer'].items():
+            ext = determine_ext(format_url)
+            print(ext)
+            if ext == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            elif ext == 'f4m':
+                f4m_formats = self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id, fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
 
         return {
             'id': video['id'],

From 2abf7cab80a2d12a3157afef05d61f8404bce45d Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 18:23:46 +0200
Subject: [PATCH 1920/2145] [soundcloud] Add Soundcloud search extractor

---
 youtube_dl/extractor/__init__.py   |  3 +-
 youtube_dl/extractor/soundcloud.py | 88 +++++++++++++++++++++++++++++-
 2 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 26e5745d6..232bcd89a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -576,7 +576,8 @@ from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
     SoundcloudUserIE,
-    SoundcloudPlaylistIE
+    SoundcloudPlaylistIE,
+    SoundcloudSearchIE
 )
 from .soundgasm import (
     SoundgasmIE,
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2b60d354a..7395a9848 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 import itertools
 
-from .common import InfoExtractor
+from .common import (
+    InfoExtractor,
+    SearchInfoExtractor
+)
 from ..compat import (
     compat_str,
     compat_urlparse,
@@ -469,3 +472,86 @@ class SoundcloudPlaylistIE(SoundcloudIE):
             'description': data.get('description'),
             'entries': entries,
         }
+
+
+class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
+    IE_NAME = 'soundcloud:search'
+    IE_DESC = 'Soundcloud search'
+    _MAX_RESULTS = 200
+    _TESTS = [{
+        'url': 'scsearch15:post-avant jazzcore',
+        'info_dict': {
+            'title': 'post-avant jazzcore',
+        },
+        'playlist_count': 15,
+    }]
+
+    _SEARCH_KEY = 'scsearch'
+    _RESULTS_PER_PAGE = 50
+
+    def _get_collection(self, endpoint, collection_id, **query):
+        import itertools
+
+        query['limit'] = self._RESULTS_PER_PAGE
+        query['client_id'] = self._CLIENT_ID
+        query['linked_partitioning'] = '1'
+
+        api_base_url = '{0}//api-v2.soundcloud.com'.format(self.http_scheme())
+
+        total_results = self._MAX_RESULTS
+        collected_results = 0
+
+        next_url = None
+
+        for i in itertools.count():
+
+            if not next_url:
+                query['offset'] = i * self._RESULTS_PER_PAGE
+                data = compat_urllib_parse.urlencode(query)
+                next_url = '{0}{1}?{2}'.format(api_base_url, endpoint, data)
+
+            response = self._download_json(next_url,
+                    video_id=collection_id,
+                    note='Downloading page {0}'.format(i+1),
+                    errnote='Unable to download API page')
+
+            total_results = int(response.get(
+                u'total_results', total_results))
+
+            collection = response['collection']
+            collected_results += len(collection)
+
+            for item in filter(bool, collection):
+                yield item
+
+            if collected_results >= total_results or not collection:
+                break
+
+            next_url = response.get(u'next_href', None)
+
+    def _get_n_results(self, query, n):
+
+        results = []
+
+        tracks = self._get_collection('/search/tracks',
+            collection_id='Query "{}"'.format(query),
+            q=query.encode('utf-8'))
+
+        for _ in range(n):
+            try:
+                track = next(tracks)
+            except StopIteration:
+                break
+            uri = track[u'uri']
+            title = track[u'title']
+            username = track[u'user'][u'username']
+            results.append(self.url_result(
+                url=uri,
+                video_title='{0} - {1}'.format(username, title)))
+
+        if not results:
+            raise ExtractorError(
+                '[soundcloud] No track results', expected=True)
+        
+        return self.playlist_result(results[:n], playlist_title=query)
+

From c30943b1c06d0f8d927031fc3a7f3cda30c4dae1 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 21:18:42 +0200
Subject: [PATCH 1921/2145] Fix some compatibility issues, cleanup.

---
 youtube_dl/extractor/soundcloud.py | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 7395a9848..3fe991849 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -488,27 +488,23 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
     _SEARCH_KEY = 'scsearch'
     _RESULTS_PER_PAGE = 50
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        import itertools
-
         query['limit'] = self._RESULTS_PER_PAGE
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
 
-        api_base_url = '{0}//api-v2.soundcloud.com'.format(self.http_scheme())
-
         total_results = self._MAX_RESULTS
         collected_results = 0
 
         next_url = None
 
         for i in itertools.count():
-
             if not next_url:
                 query['offset'] = i * self._RESULTS_PER_PAGE
                 data = compat_urllib_parse.urlencode(query)
-                next_url = '{0}{1}?{2}'.format(api_base_url, endpoint, data)
+                next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
 
             response = self._download_json(next_url,
                     video_id=collection_id,
@@ -516,7 +512,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
                     errnote='Unable to download API page')
 
             total_results = int(response.get(
-                u'total_results', total_results))
+                'total_results', total_results))
 
             collection = response['collection']
             collected_results += len(collection)
@@ -527,14 +523,13 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             if collected_results >= total_results or not collection:
                 break
 
-            next_url = response.get(u'next_href', None)
+            next_url = response.get('next_href', None)
 
     def _get_n_results(self, query, n):
-
         results = []
 
         tracks = self._get_collection('/search/tracks',
-            collection_id='Query "{}"'.format(query),
+            collection_id='Query "{0}"'.format(query),
             q=query.encode('utf-8'))
 
         for _ in range(n):
@@ -542,12 +537,9 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
                 track = next(tracks)
             except StopIteration:
                 break
-            uri = track[u'uri']
-            title = track[u'title']
-            username = track[u'user'][u'username']
-            results.append(self.url_result(
-                url=uri,
-                video_title='{0} - {1}'.format(username, title)))
+            uri = track['uri']
+            title = track['title']
+            results.append(self.url_result(url=uri))
 
         if not results:
             raise ExtractorError(

From b54b08c91bb7f59a9ea720abbf944809d1ea8956 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 22:36:08 +0200
Subject: [PATCH 1922/2145] Simplify with itertools.islice().

---
 youtube_dl/extractor/soundcloud.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3fe991849..959f27975 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -532,11 +532,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             collection_id='Query "{0}"'.format(query),
             q=query.encode('utf-8'))
 
-        for _ in range(n):
-            try:
-                track = next(tracks)
-            except StopIteration:
-                break
+        for track in itertools.islice(tracks, n):
             uri = track['uri']
             title = track['title']
             results.append(self.url_result(url=uri))

From 6ea7190a3e6165f3d5c49e1e142eb85951b44712 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 22:47:16 +0200
Subject: [PATCH 1923/2145] Rewrite as list comprehension.

---
 youtube_dl/extractor/soundcloud.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 959f27975..c623bb6de 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -526,16 +526,12 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             next_url = response.get('next_href', None)
 
     def _get_n_results(self, query, n):
-        results = []
-
         tracks = self._get_collection('/search/tracks',
             collection_id='Query "{0}"'.format(query),
             q=query.encode('utf-8'))
 
-        for track in itertools.islice(tracks, n):
-            uri = track['uri']
-            title = track['title']
-            results.append(self.url_result(url=uri))
+        results = [self.url_result(url=track['uri'])
+            for track in itertools.islice(tracks, n)]
 
         if not results:
             raise ExtractorError(

From 417b4536999daaae2e2cc105012cd1c1e8046c15 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sun, 18 Oct 2015 12:43:25 +0200
Subject: [PATCH 1924/2145] [soundcloud] Use correct error message conventions

---
 youtube_dl/extractor/soundcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c623bb6de..6b510a43b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -535,7 +535,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
         if not results:
             raise ExtractorError(
-                '[soundcloud] No track results', expected=True)
+                'Soundcloud said: No track results', expected=True)
         
-        return self.playlist_result(results[:n], playlist_title=query)
+        return self.playlist_result(results, playlist_title=query)
 

From 328a22e175a4e056258d50ebffe28026b9411285 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Fri, 30 Oct 2015 23:56:07 +0100
Subject: [PATCH 1925/2145] [soundcloud] Remove limit on search results

---
 youtube_dl/extractor/soundcloud.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 6b510a43b..bba29d845 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -477,7 +477,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     IE_NAME = 'soundcloud:search'
     IE_DESC = 'Soundcloud search'
-    _MAX_RESULTS = 200
+    _MAX_RESULTS = float('inf')
     _TESTS = [{
         'url': 'scsearch15:post-avant jazzcore',
         'info_dict': {
@@ -487,24 +487,28 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     }]
 
     _SEARCH_KEY = 'scsearch'
-    _RESULTS_PER_PAGE = 50
+    _MAX_RESULTS_PER_PAGE = 200
+    _DEFAULT_RESULTS_PER_PAGE = 50
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        query['limit'] = self._RESULTS_PER_PAGE
+        query['limit'] = results_per_page = min(
+            query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
+            self._MAX_RESULTS_PER_PAGE)
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
 
-        total_results = self._MAX_RESULTS
+        total_results = None
         collected_results = 0
 
         next_url = None
 
         for i in itertools.count():
             if not next_url:
-                query['offset'] = i * self._RESULTS_PER_PAGE
+                query['offset'] = i * results_per_page
                 data = compat_urllib_parse.urlencode(query)
-                next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
+                next_url = '{0}{1}?{2}'.format(
+                    self._API_V2_BASE, endpoint, data)
 
             response = self._download_json(next_url,
                     video_id=collection_id,
@@ -520,7 +524,8 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             for item in filter(bool, collection):
                 yield item
 
-            if collected_results >= total_results or not collection:
+            if (total_results is not None and
+                collected_results >= total_results) or not collection:
                 break
 
             next_url = response.get('next_href', None)
@@ -528,7 +533,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     def _get_n_results(self, query, n):
         tracks = self._get_collection('/search/tracks',
             collection_id='Query "{0}"'.format(query),
-            q=query.encode('utf-8'))
+            limit=n, q=query)
 
         results = [self.url_result(url=track['uri'])
             for track in itertools.islice(tracks, n)]

From 7e3472758bfaa75aa413368b29a26c2615e5231b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:04:35 +0600
Subject: [PATCH 1926/2145] [soundcloud:search] PEP 8

---
 youtube_dl/extractor/soundcloud.py | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index bba29d845..a5c40514b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -510,10 +510,9 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
                 next_url = '{0}{1}?{2}'.format(
                     self._API_V2_BASE, endpoint, data)
 
-            response = self._download_json(next_url,
-                    video_id=collection_id,
-                    note='Downloading page {0}'.format(i+1),
-                    errnote='Unable to download API page')
+            response = self._download_json(
+                next_url, collection_id, 'Downloading page {0}'.format(i + 1),
+                'Unable to download API page')
 
             total_results = int(response.get(
                 'total_results', total_results))
@@ -524,23 +523,19 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             for item in filter(bool, collection):
                 yield item
 
-            if (total_results is not None and
-                collected_results >= total_results) or not collection:
+            if (total_results is not None and collected_results >= total_results) or not collection:
                 break
 
-            next_url = response.get('next_href', None)
+            next_url = response.get('next_href')
 
     def _get_n_results(self, query, n):
-        tracks = self._get_collection('/search/tracks',
-            collection_id='Query "{0}"'.format(query),
-            limit=n, q=query)
+        tracks = self._get_collection(
+            '/search/tracks', collection_id='Query "{0}"'.format(query), limit=n, q=query)
 
-        results = [self.url_result(url=track['uri'])
-            for track in itertools.islice(tracks, n)]
+        results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
 
         if not results:
             raise ExtractorError(
                 'Soundcloud said: No track results', expected=True)
-        
-        return self.playlist_result(results, playlist_title=query)
 
+        return self.playlist_result(results, playlist_title=query)

From 7e1f5447e76e57af58bf45ce565742c813c80b99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:45:50 +0600
Subject: [PATCH 1927/2145] [utils] Improve encode_dict

---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d00b14b86..bff59eb73 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1668,7 +1668,9 @@ def urlencode_postdata(*args, **kargs):
 
 
 def encode_dict(d, encoding='utf-8'):
-    return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
+    def encode(v):
+        return v.encode(encoding) if isinstance(v, compat_basestring) else v
+    return dict((encode(k), encode(v)) for k, v in d.items())
 
 
 US_RATINGS = {

From 4e3b3030168927bd589343c51278040a5cec6203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:55:48 +0600
Subject: [PATCH 1928/2145] [soundcloud:search] Fix non-ASCII searches

---
 youtube_dl/extractor/soundcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index a5c40514b..66bb1cb54 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -14,6 +14,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     int_or_none,
     unified_strdate,
@@ -506,7 +507,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
         for i in itertools.count():
             if not next_url:
                 query['offset'] = i * results_per_page
-                data = compat_urllib_parse.urlencode(query)
+                data = compat_urllib_parse.urlencode(encode_dict(query))
                 next_url = '{0}{1}?{2}'.format(
                     self._API_V2_BASE, endpoint, data)
 

From 7dc011c063c5c51712605ac08cf1c01841d5867c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:00:42 +0600
Subject: [PATCH 1929/2145] [soundcloud:search] Remove no track results message

---
 youtube_dl/extractor/soundcloud.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 66bb1cb54..e880872ae 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -535,8 +535,4 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
         results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
 
-        if not results:
-            raise ExtractorError(
-                'Soundcloud said: No track results', expected=True)
-
         return self.playlist_result(results, playlist_title=query)

From f6c903e708bf8d9156ce5b999a220c4480a7aca5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:21:21 +0600
Subject: [PATCH 1930/2145] [soundcloud:search] Simplify (Closes #7213)

---
 youtube_dl/extractor/soundcloud.py | 42 +++++++++++++-----------------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e880872ae..9b252c444 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -493,46 +493,40 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        query['limit'] = results_per_page = min(
+        limit = results_per_page = min(
             query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
             self._MAX_RESULTS_PER_PAGE)
+        query['limit'] = limit
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
+        query['offset'] = 0
+        data = compat_urllib_parse.urlencode(encode_dict(query))
+        next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
 
-        total_results = None
         collected_results = 0
 
-        next_url = None
-
-        for i in itertools.count():
-            if not next_url:
-                query['offset'] = i * results_per_page
-                data = compat_urllib_parse.urlencode(encode_dict(query))
-                next_url = '{0}{1}?{2}'.format(
-                    self._API_V2_BASE, endpoint, data)
-
+        for i in itertools.count(1):
             response = self._download_json(
-                next_url, collection_id, 'Downloading page {0}'.format(i + 1),
+                next_url, collection_id, 'Downloading page {0}'.format(i),
                 'Unable to download API page')
 
-            total_results = int(response.get(
-                'total_results', total_results))
+            collection = response.get('collection', [])
+            if not collection:
+                break
 
-            collection = response['collection']
+            collection = list(filter(bool, collection))
             collected_results += len(collection)
 
-            for item in filter(bool, collection):
-                yield item
+            for item in collection:
+                yield self.url_result(item['uri'], SoundcloudIE.ie_key())
 
-            if (total_results is not None and collected_results >= total_results) or not collection:
+            if not collection or collected_results >= limit:
                 break
 
             next_url = response.get('next_href')
+            if not next_url:
+                break
 
     def _get_n_results(self, query, n):
-        tracks = self._get_collection(
-            '/search/tracks', collection_id='Query "{0}"'.format(query), limit=n, q=query)
-
-        results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
-
-        return self.playlist_result(results, playlist_title=query)
+        tracks = self._get_collection('/search/tracks', query, limit=n, q=query)
+        return self.playlist_result(tracks, playlist_title=query)

From c3a227d1c406d0af8fdf6afd9e33366e903d9a8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:25:48 +0600
Subject: [PATCH 1931/2145] [pluralsight] Update _LOGIN_URL

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 93244f41d..417dd965c 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -19,7 +19,7 @@ from ..utils import (
 class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
-    _LOGIN_URL = 'https://www.pluralsight.com/id/'
+    _LOGIN_URL = 'https://app.pluralsight.com/id/'
     _NETRC_MACHINE = 'pluralsight'
 
     _TESTS = [{

From 0533915aad95d76b6bf4f529d39167f22a7c4b90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:35:08 +0600
Subject: [PATCH 1932/2145] [pluralsight] Update some more URLs

---
 youtube_dl/extractor/pluralsight.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 417dd965c..fe6850aac 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -20,6 +20,7 @@ class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
     _LOGIN_URL = 'https://app.pluralsight.com/id/'
+    _API_BASE = 'http://app.pluralsight.com'
     _NETRC_MACHINE = 'pluralsight'
 
     _TESTS = [{
@@ -142,7 +143,7 @@ class PluralsightIE(InfoExtractor):
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
                 request = compat_urllib_request.Request(
-                    'http://app.pluralsight.com/training/Player/ViewClip',
+                    '%s/training/Player/ViewClip' % self._API_BASE,
                     json.dumps(clip_post).encode('utf-8'))
                 request.add_header('Content-Type', 'application/json;charset=utf-8')
                 format_id = '%s-%s' % (ext, quality)
@@ -201,14 +202,14 @@ class PluralsightCourseIE(InfoExtractor):
         # TODO: PSM cookie
 
         course = self._download_json(
-            'http://www.pluralsight.com/data/course/%s' % course_id,
+            '%s/data/course/%s' % (self._API_BASE, course_id),
             course_id, 'Downloading course JSON')
 
         title = course['title']
         description = course.get('description') or course.get('shortDescription')
 
         course_data = self._download_json(
-            'http://www.pluralsight.com/data/course/content/%s' % course_id,
+            '%s/data/course/content/%s' % (self._API_BASE, course_id),
             course_id, 'Downloading course data JSON')
 
         entries = []
@@ -218,7 +219,7 @@ class PluralsightCourseIE(InfoExtractor):
                 if not player_parameters:
                     continue
                 entries.append(self.url_result(
-                    'http://www.pluralsight.com/training/player?%s' % player_parameters,
+                    '%s/training/player?%s' % (self._API_BASE, player_parameters),
                     'Pluralsight'))
 
         return self.playlist_result(entries, course_id, title, description)

From 563772eda4a42b1fce1f3740fe91bf74e63bd347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:37:29 +0600
Subject: [PATCH 1933/2145] [pluralsight] Extract base class

---
 youtube_dl/extractor/pluralsight.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index fe6850aac..de7dc739b 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -16,11 +16,15 @@ from ..utils import (
 )
 
 
-class PluralsightIE(InfoExtractor):
+class PluralsightBaseIE(InfoExtractor):
+    _API_BASE = 'http://app.pluralsight.com'
+
+
+class PluralsightIE(PluralsightBaseIE):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
     _LOGIN_URL = 'https://app.pluralsight.com/id/'
-    _API_BASE = 'http://app.pluralsight.com'
+
     _NETRC_MACHINE = 'pluralsight'
 
     _TESTS = [{
@@ -174,7 +178,7 @@ class PluralsightIE(InfoExtractor):
         }
 
 
-class PluralsightCourseIE(InfoExtractor):
+class PluralsightCourseIE(PluralsightBaseIE):
     IE_NAME = 'pluralsight:course'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
     _TESTS = [{

From 7e508ff2cf8f8f8b1784db6fb33994839841d122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:49:37 +0600
Subject: [PATCH 1934/2145] [pluralsight] Improve login detection

---
 youtube_dl/extractor/pluralsight.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index de7dc739b..7693282a5 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -84,6 +84,9 @@ class PluralsightIE(PluralsightBaseIE):
         if error:
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
+        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
+            raise ExtractorError('Unable to log in')
+
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
 

From bea56c95699af594586095e5ea88e9857049c6a1 Mon Sep 17 00:00:00 2001
From: Andrzej Lichnerowicz <andrzej@lichnerowicz.pl>
Date: Sat, 26 Sep 2015 21:58:33 +0200
Subject: [PATCH 1935/2145] [pluralsight] prevent error 429 when sensing video
 formats

---
 youtube_dl/extractor/pluralsight.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7693282a5..d542a9e0e 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -156,6 +156,9 @@ class PluralsightIE(PluralsightBaseIE):
                 format_id = '%s-%s' % (ext, quality)
                 clip_url = self._download_webpage(
                     request, display_id, 'Downloading %s URL' % format_id, fatal=False)
+                # #6989: sleep 3 seconds to avoid 429 errors.
+                # should help with #6842.
+                self._sleep(3, display_id)
                 if not clip_url:
                     continue
                 f.update({

From 38eb2968abce9078f28904046b8e26b0d87ada61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:07:09 +0600
Subject: [PATCH 1936/2145] [pluralsight] Clarify and randomize ViewClip sleep
 interval

---
 youtube_dl/extractor/pluralsight.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index d542a9e0e..8481a10b0 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import json
+import random
 
 from .common import InfoExtractor
 from ..compat import (
@@ -156,9 +157,17 @@ class PluralsightIE(PluralsightBaseIE):
                 format_id = '%s-%s' % (ext, quality)
                 clip_url = self._download_webpage(
                     request, display_id, 'Downloading %s URL' % format_id, fatal=False)
-                # #6989: sleep 3 seconds to avoid 429 errors.
-                # should help with #6842.
-                self._sleep(3, display_id)
+
+                # Pluralsight tracks multiple sequential calls to ViewClip API and start
+                # to return 429 HTTP errors after some time (see
+                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
+                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
+                # To somewhat reduce the probability of these consequences
+                # we will sleep random amount of time before each call to ViewClip.
+                self._sleep(
+                    random.randint(2, 5), display_id,
+                    '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')
+
                 if not clip_url:
                     continue
                 f.update({

From 4c57b4853d8c1b41d729314e316cc1b83129ea18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:42:58 +0600
Subject: [PATCH 1937/2145] [pluralsight] Until listing formats request only
 single format

---
 youtube_dl/extractor/pluralsight.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 8481a10b0..16de0fd1c 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import json
 import random
+import collections
 
 from .common import InfoExtractor
 from ..compat import (
@@ -131,13 +132,30 @@ class PluralsightIE(PluralsightBaseIE):
             'high': {'width': 1024, 'height': 768},
         }
 
+        AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
+
         ALLOWED_QUALITIES = (
-            ('webm', ('high',)),
-            ('mp4', ('low', 'medium', 'high',)),
+            AllowedQuality('webm', ('high',)),
+            AllowedQuality('mp4', ('low', 'medium', 'high',)),
         )
 
+        if self._downloader.params.get('listformats', False):
+            allowed_qualities = ALLOWED_QUALITIES
+        else:
+            def guess_allowed_qualities():
+                req_format = self._downloader.params.get('format') or 'best'
+                req_format_split = req_format.split('-')
+                if len(req_format_split) > 1:
+                    req_ext, req_quality = req_format_split
+                    for allowed_quality in ALLOWED_QUALITIES:
+                        if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
+                            return (AllowedQuality(req_ext, (req_quality, )), )
+                req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
+                return (AllowedQuality(req_ext, ('high', )), )
+            allowed_qualities = guess_allowed_qualities()
+
         formats = []
-        for ext, qualities in ALLOWED_QUALITIES:
+        for ext, qualities in allowed_qualities:
             for quality in qualities:
                 f = QUALITIES[quality].copy()
                 clip_post = {

From a3372437bfe57f5b450bfdd60105bb6120595928 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:49:58 +0600
Subject: [PATCH 1938/2145] [soundcloud] Remove unused variable

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 9b252c444..02e64e094 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -493,7 +493,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        limit = results_per_page = min(
+        limit = min(
             query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
             self._MAX_RESULTS_PER_PAGE)
         query['limit'] = limit

From cf186b77a79a0bf6f322bc1e2f610471eab3b01c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:56:40 +0600
Subject: [PATCH 1939/2145] [pluralsight] Clarify allowed qualities guessing
 rationale

---
 youtube_dl/extractor/pluralsight.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 16de0fd1c..3dee616c1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -139,6 +139,9 @@ class PluralsightIE(PluralsightBaseIE):
             AllowedQuality('mp4', ('low', 'medium', 'high',)),
         )
 
+        # In order to minimize the number of calls to ViewClip API and reduce
+        # the probability of being throttled or banned by Pluralsight we will request
+        # only single format until explicit listformats was requested.
         if self._downloader.params.get('listformats', False):
             allowed_qualities = ALLOWED_QUALITIES
         else:

From 0eebf34d9dd793bce9148dae267d690d50d89f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:58:25 +0600
Subject: [PATCH 1940/2145] [pluralsight] Rephrase

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 3dee616c1..a11cebaf1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -141,7 +141,7 @@ class PluralsightIE(PluralsightBaseIE):
 
         # In order to minimize the number of calls to ViewClip API and reduce
         # the probability of being throttled or banned by Pluralsight we will request
-        # only single format until explicit listformats was requested.
+        # only single format until formats listing was explicitly requested.
         if self._downloader.params.get('listformats', False):
             allowed_qualities = ALLOWED_QUALITIES
         else:

From 0c14841585db1baa2f9a4a5ff263035977cf0964 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 04:17:07 +0600
Subject: [PATCH 1941/2145] [youtube:user:playlists] Add extractor (Closes
 #3817)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/youtube.py  | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 232bcd89a..947b83683 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -834,6 +834,7 @@ from .youtube import (
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
+    YoutubeUserPlaylistsIE,
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 364ca102a..abc67f07f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -224,6 +224,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
         return zip(ids_in_page, titles_in_page)
 
 
+class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        entries = [
+            self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
+            for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
+        title = self._og_search_title(webpage, fatal=False)
+        return self.playlist_result(entries, playlist_id, title)
+
+
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
     _VALID_URL = r"""(?x)^
@@ -1742,6 +1753,21 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
+class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
+    IE_DESC = 'YouTube.com user playlists'
+    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
+    IE_NAME = 'youtube:user:playlists'
+
+    _TEST = {
+        'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
+        'playlist_mincount': 4,
+        'info_dict': {
+            'id': 'ThirstForScience',
+            'title': 'Thirst for Science',
+        },
+    }
+
+
 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
     IE_DESC = 'YouTube.com searches'
     # there doesn't appear to be a real limit, for example if you search for

From 136dadde9543a80f490b26c822dcfdff5541c335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 04:18:20 +0600
Subject: [PATCH 1942/2145] [youtube:show] Rework in terms of playlists base
 extractor

---
 youtube_dl/extractor/youtube.py | 25 ++++---------------------
 1 file changed, 4 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index abc67f07f..c56f8a0a2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1863,7 +1863,7 @@ class YoutubeSearchURLIE(InfoExtractor):
         }
 
 
-class YoutubeShowIE(InfoExtractor):
+class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
     IE_DESC = 'YouTube.com (multi-season) shows'
     _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
     IE_NAME = 'youtube:show'
@@ -1877,26 +1877,9 @@ class YoutubeShowIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
-        webpage = self._download_webpage(
-            'https://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
-        # There's one playlist for each season of the show
-        m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
-        self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))
-        entries = [
-            self.url_result(
-                'https://www.youtube.com' + season.group(1), 'YoutubePlaylist')
-            for season in m_seasons
-        ]
-        title = self._og_search_title(webpage, fatal=False)
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': title,
-            'entries': entries,
-        }
+        playlist_id = self._match_id(url)
+        return super(YoutubeShowIE, self)._real_extract(
+            'https://www.youtube.com/show/%s/playlists' % playlist_id)
 
 
 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):

From 82c4d7b0ce49b13a9dd8491dd32e962ca307c974 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Nov 2015 23:36:27 +0100
Subject: [PATCH 1943/2145] release 2015.11.21

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f56a6986b..1df408610 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -494,6 +494,7 @@
  - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
+ - **soundcloud:search**: Soundcloud search
  - **soundcloud:set**
  - **soundcloud:user**
  - **soundgasm**
@@ -707,6 +708,7 @@
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
+ - **youtube:user:playlists**: YouTube.com user playlists
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b3b1b90ba..2baf1ac42 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.19'
+__version__ = '2015.11.21'

From 061a75edd6bab2f30978b458fe7402ff9e9c02a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:01:01 +0600
Subject: [PATCH 1944/2145] [youtube] Extract base for entry list extractors
 and support multi page lists of playlists

---
 youtube_dl/extractor/youtube.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c56f8a0a2..8352ad1da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,15 +178,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
-class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
-    # Extract the video ids from the playlist pages
+class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
+    # Extract entries from page with "Load more" button
     def _entries(self, page, playlist_id):
         more_widget_html = content_html = page
         for page_num in itertools.count(1):
-            for video_id, video_title in self.extract_videos_from_page(content_html):
-                yield self.url_result(
-                    video_id, 'Youtube', video_id=video_id,
-                    video_title=video_title)
+            for entry in self._process_page(content_html):
+                yield entry
 
             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
             if not mobj:
@@ -203,6 +201,12 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
                 break
             more_widget_html = more['load_more_widget_html']
 
+
+class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+    def _process_page(self, content):
+        for video_id, video_title in self.extract_videos_from_page(content):
+            yield self.url_result(video_id, 'Youtube', video_id, video_title)
+
     def extract_videos_from_page(self, page):
         ids_in_page = []
         titles_in_page = []
@@ -224,15 +228,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
         return zip(ids_in_page, titles_in_page)
 
 
-class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+    def _process_page(self, content):
+        for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
+            yield self.url_result(
+                'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
-        entries = [
-            self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
-            for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
         title = self._og_search_title(webpage, fatal=False)
-        return self.playlist_result(entries, playlist_id, title)
+        return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
 
 
 class YoutubeIE(YoutubeBaseInfoExtractor):

From e568c2233e7b4b27c9a5c56322ab7633a5f0b1f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:03:23 +0600
Subject: [PATCH 1945/2145] [youtube] Add test for multi page list of playlists

---
 youtube_dl/extractor/youtube.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8352ad1da..4a0ff6e9c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1764,14 +1764,22 @@ class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
     IE_NAME = 'youtube:user:playlists'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
         'playlist_mincount': 4,
         'info_dict': {
             'id': 'ThirstForScience',
             'title': 'Thirst for Science',
         },
-    }
+    }, {
+        # with "Load more" button
+        'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
+        'playlist_mincount': 70,
+        'info_dict': {
+            'id': 'igorkle1',
+            'title': 'Игорь Клейнер',
+        },
+    }]
 
 
 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):

From 3e12bc583af9d5abf5f144ed6e092c59f4b83fdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 06:29:39 +0600
Subject: [PATCH 1946/2145] [utils] Improve determine_ext (Closes #7593)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bff59eb73..7dab60bb8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -922,7 +922,7 @@ def unified_strdate(date_str, day_first=True):
 def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
-    guess = url.partition('?')[0].rpartition('.')[2]
+    guess = url.partition('?')[0].rpartition('.')[2].rstrip('/')
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
     else:

From 5035536e3f32d4c47b2d3067c12e074cb9a4a199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 06:33:52 +0600
Subject: [PATCH 1947/2145] [test_utils] Add tests for determine_ext

---
 test/test_utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index ea1ff0547..4f0d9e481 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -21,6 +21,7 @@ from youtube_dl.utils import (
     clean_html,
     DateRange,
     detect_exe_version,
+    determine_ext,
     encodeFilename,
     escape_rfc3986,
     escape_url,
@@ -238,6 +239,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
+    def test_determine_ext(self):
+        self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
+        self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
+
     def test_find_xpath_attr(self):
         testxml = '''<root>
             <node/>

From 9cb9a5df7794579c38efff1c4b1451a7d13da3c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 17:27:13 +0600
Subject: [PATCH 1948/2145] [utils] Check ext with trailing slash against the
 list of known extensions

---
 test/test_utils.py  |  3 +++
 youtube_dl/utils.py | 17 ++++++++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4f0d9e481..501355c74 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -242,6 +242,9 @@ class TestUtil(unittest.TestCase):
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
         self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
+        self.assertEqual(determine_ext('http://example.com/foo/bar.nonext/?download', None), None)
+        self.assertEqual(determine_ext('http://example.com/foo/bar/mp4?download', None), None)
+        self.assertEqual(determine_ext('http://example.com/foo/bar.m3u8//?download'), 'm3u8')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7dab60bb8..c0325f054 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -922,9 +922,24 @@ def unified_strdate(date_str, day_first=True):
 def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
-    guess = url.partition('?')[0].rpartition('.')[2].rstrip('/')
+    guess = url.partition('?')[0].rpartition('.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
+    elif guess.rstrip('/') in (
+            'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
+            'flv', 'f4v', 'f4a', 'f4b',
+            'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
+            'mkv', 'mka', 'mk3d',
+            'avi', 'divx',
+            'mov',
+            'asf', 'wmv', 'wma',
+            '3gp', '3g2',
+            'mp3',
+            'flac',
+            'ape',
+            'wav',
+            'f4f', 'f4m', 'm3u8', 'smil'):
+        return guess.rstrip('/')
     else:
         return default_ext
 

From 1b38185361e096d6e34db11adac7333ac9dadca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 18:08:30 +0600
Subject: [PATCH 1949/2145] [pornhd] Fix title extraction (Closes #7596)

---
 youtube_dl/extractor/pornhd.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index dbb2c3bd9..57c78ba52 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -36,7 +36,8 @@ class PornHdIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id or video_id)
 
         title = self._html_search_regex(
-            r'<title>(.+) porn HD.+?</title>', webpage, 'title')
+            [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
+             r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
         description = self._html_search_regex(
             r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
         view_count = int_or_none(self._html_search_regex(

From 3cfd000849208b58dab4f78d1486d3f24552009e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:14:35 +0100
Subject: [PATCH 1950/2145] [youtube] More explicit player config JSON
 extraction (fixes #7468)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 687e0b4db..21731188a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1074,7 +1074,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             age_gate = False
             video_info = None
             # Try looking directly into the video webpage
-            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});ytplayer', video_webpage)
             if mobj:
                 json_code = uppercase_escape(mobj.group(1))
                 ytplayer_config = json.loads(json_code)

From 4a7d108ab3c8b5ac82b8740179dae6a454218a38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 18:24:17 +0600
Subject: [PATCH 1951/2145] [rutube] Remove unnecessary print

---
 youtube_dl/extractor/rutube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 0db8c410d..6b09550b0 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -54,7 +54,6 @@ class RutubeIE(InfoExtractor):
         formats = []
         for format_id, format_url in options['video_balancer'].items():
             ext = determine_ext(format_url)
-            print(ext)
             if ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
                     format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)

From 0e49d9a6b0216555c2a3ee063ae3d1c6d09edbd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:49:33 +0100
Subject: [PATCH 1952/2145] [youtube] Fall back to the original regex for
 ytplayer.config

---
 youtube_dl/extractor/youtube.py | 39 ++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 21731188a..7e74d2368 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -674,7 +674,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         {
             'url': 'http://vid.plus/FlRa-iH7PGw',
             'only_matching': True,
-        }
+        },
+        {
+            # Title with JS-like syntax "};"
+            'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
+            'info_dict': {
+                'id': 'lsguqyKfVQg',
+                'ext': 'mp4',
+                'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
+                'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
+                'upload_date': '20151119',
+                'uploader_id': 'IronSoulElf',
+                'uploader': 'IronSoulElf',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -858,16 +874,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return {}
         return sub_lang_list
 
+    def _get_ytplayer_config(self, webpage):
+        patterns = [
+            r';ytplayer\.config\s*=\s*({.*?});ytplayer',
+            r';ytplayer\.config\s*=\s*({.*?});',
+        ]
+        for pattern in patterns:
+            config = self._search_regex(pattern, webpage, 'ytconfig.player', default=None)
+            if config is not None:
+                return json.loads(uppercase_escape(config))
+
     def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
         self.to_screen('%s: Looking for automatic captions' % video_id)
-        mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
+        player_config = self._get_ytplayer_config(webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
-        if mobj is None:
+        if player_config is None:
             self._downloader.report_warning(err_msg)
             return {}
-        player_config = json.loads(mobj.group(1))
         try:
             args = player_config['args']
             caption_url = args['ttsurl']
@@ -1074,10 +1099,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             age_gate = False
             video_info = None
             # Try looking directly into the video webpage
-            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});ytplayer', video_webpage)
-            if mobj:
-                json_code = uppercase_escape(mobj.group(1))
-                ytplayer_config = json.loads(json_code)
+            ytplayer_config = self._get_ytplayer_config(video_webpage)
+            if ytplayer_config is not None:
                 args = ytplayer_config['args']
                 if args.get('url_encoded_fmt_stream_map'):
                     # Convert to the same format returned by compat_parse_qs

From b41631c4e6e56afb2427513c84df1b13681cf4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:53:26 +0100
Subject: [PATCH 1953/2145] [youtube] Send the list of patterns directly to
 _search_regex

---
 youtube_dl/extractor/youtube.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7e74d2368..247769067 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -879,10 +879,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             r';ytplayer\.config\s*=\s*({.*?});ytplayer',
             r';ytplayer\.config\s*=\s*({.*?});',
         ]
-        for pattern in patterns:
-            config = self._search_regex(pattern, webpage, 'ytconfig.player', default=None)
-            if config is not None:
-                return json.loads(uppercase_escape(config))
+        config = self._search_regex(patterns, webpage, 'ytconfig.player', default=None)
+        if config is not None:
+            return json.loads(uppercase_escape(config))
 
     def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an

From 02f0da20b00fa0efee6abcd1842c183b3a4cc36c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 03:08:38 +0600
Subject: [PATCH 1954/2145] [pluralsight] Add support for alternative webpage
 layout (Closes #7607)

---
 youtube_dl/extractor/pluralsight.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index a11cebaf1..792316db8 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -104,19 +104,29 @@ class PluralsightIE(PluralsightBaseIE):
 
         webpage = self._download_webpage(url, display_id)
 
-        collection = self._parse_json(
-            self._search_regex(
-                r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
-                webpage, 'modules'),
-            display_id)
+        modules = self._search_regex(
+            r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
+            webpage, 'modules', default=None)
+
+        if modules:
+            collection = self._parse_json(modules, display_id)
+        else:
+            # Webpage may be served in different layout (see
+            # https://github.com/rg3/youtube-dl/issues/7607)
+            collection = self._parse_json(
+                self._search_regex(
+                    r'var\s+initialState\s*=\s*({.+?});\n', webpage, 'initial state'),
+                display_id)['course']['modules']
 
         module, clip = None, None
 
         for module_ in collection:
-            if module_.get('moduleName') == name:
+            if name in (module_.get('moduleName'), module_.get('name')):
                 module = module_
                 for clip_ in module_.get('clips', []):
                     clip_index = clip_.get('clipIndex')
+                    if clip_index is None:
+                        clip_index = clip_.get('index')
                     if clip_index is None:
                         continue
                     if compat_str(clip_index) == clip_id:

From a72778d364022612ba88bdfd9affef0d7b0ca864 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:00:06 +0600
Subject: [PATCH 1955/2145] [youtube] Improve ytplayer.config extraction

---
 youtube_dl/extractor/youtube.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1580c54fe..052f6922a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -891,22 +891,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return {}
         return sub_lang_list
 
-    def _get_ytplayer_config(self, webpage):
-        patterns = [
-            r';ytplayer\.config\s*=\s*({.*?});ytplayer',
-            r';ytplayer\.config\s*=\s*({.*?});',
-        ]
-        config = self._search_regex(patterns, webpage, 'ytconfig.player', default=None)
-        if config is not None:
-            return json.loads(uppercase_escape(config))
+    def _get_ytplayer_config(self, video_id, webpage):
+        patterns = (
+            r';ytplayer\.config\s*=\s*({.+?});ytplayer',
+            r';ytplayer\.config\s*=\s*({.+?});',
+        )
+        config = self._search_regex(
+            patterns, webpage, 'ytplayer.config', default=None)
+        if config:
+            return self._parse_json(
+                uppercase_escape(config), video_id, fatal=False)
 
     def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
         self.to_screen('%s: Looking for automatic captions' % video_id)
-        player_config = self._get_ytplayer_config(webpage)
+        player_config = self._get_ytplayer_config(video_id, webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
-        if player_config is None:
+        if not player_config:
             self._downloader.report_warning(err_msg)
             return {}
         try:
@@ -1115,8 +1117,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             age_gate = False
             video_info = None
             # Try looking directly into the video webpage
-            ytplayer_config = self._get_ytplayer_config(video_webpage)
-            if ytplayer_config is not None:
+            ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
+            if ytplayer_config:
                 args = ytplayer_config['args']
                 if args.get('url_encoded_fmt_stream_map'):
                     # Convert to the same format returned by compat_parse_qs

From 61f92af1cfacb9a5a6e368d0093fb71dbac0af6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:02:37 +0600
Subject: [PATCH 1956/2145] [youtube] Add test with '};' in tags

---
 youtube_dl/extractor/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 052f6922a..824335d0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -693,7 +693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'only_matching': True,
         },
         {
-            # Title with JS-like syntax "};"
+            # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
             'info_dict': {
                 'id': 'lsguqyKfVQg',
@@ -708,6 +708,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
+            'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
+            'only_matching': True,
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From 526b3b071632bc3c840ae4dd3579e015f41df6f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:14:03 +0600
Subject: [PATCH 1957/2145] [youtube] Clarify ytplayer.config extraction
 rationale

---
 youtube_dl/extractor/youtube.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 824335d0a..5482aac3b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -898,6 +898,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _get_ytplayer_config(self, video_id, webpage):
         patterns = (
+            # User data may contain arbitrary character sequences that may affect
+            # JSON extraction with regex, e.g. when '};' is contained the second
+            # regex won't capture the whole JSON. Yet working around by trying more
+            # concrete regex first keeping in mind proper quoted string handling
+            # to be implemented in future that will replace this workaround (see
+            # https://github.com/rg3/youtube-dl/issues/7468,
+            # https://github.com/rg3/youtube-dl/pull/7599)
             r';ytplayer\.config\s*=\s*({.+?});ytplayer',
             r';ytplayer\.config\s*=\s*({.+?});',
         )

From 94bfcd23b7e2488b5e4bbf076965ab9ed980f1ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:35:23 +0600
Subject: [PATCH 1958/2145] [youtube] Fix test

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5482aac3b..e0e345496 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -426,7 +426,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                 'uploader': 'SET India',
-                'uploader_id': 'setindia'
+                'uploader_id': 'setindia',
+                'age_limit': 18,
             }
         },
         {

From 9022726446c659f2bc38556105991e9797e0c8c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:37:21 +0600
Subject: [PATCH 1959/2145] [youtube] Fix test

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e0e345496..0246050c2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -564,7 +564,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
-                'upload_date': '20120724',
+                'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',

From 13a10d5aa336be7c301a6d09eb4e9d7b50f51191 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:08:01 +0600
Subject: [PATCH 1960/2145] [compat] Add compat_urllib_request_Request

This is actually not a compatibility routine but rather a workaround for URLs without protocol specified.
The protocol-less URL is treated as HTTP one since it's most probable scenario and it will most likely to
redirect to HTTPS if HTTPS was actually expected. This routine could also be useful for any Request
preprocessing that may be added in future.
---
 youtube_dl/compat.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3e85264a..4e3de7f51 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,6 +198,14 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
+
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def compat_urllib_request_Request(url, *args, **kwargs):
+    return compat_urllib_request.Request(
+        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
 try:
     compat_basestring = basestring  # Python 2
 except NameError:

From 82d8a8b6e2eadb2c1ac8684ce9876f40925f4e2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:08:34 +0600
Subject: [PATCH 1961/2145] [YoutubeDL] Wrap plain-text URL requests in
 compat_urllib_request_Request

---
 youtube_dl/YoutubeDL.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fba99af8d..b20837ce2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,6 +28,7 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
+    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_get_terminal_size,
@@ -38,6 +39,7 @@ from .compat import (
     compat_urllib_error,
     compat_urllib_request,
     compat_urllib_request_DataHandler,
+    compat_urllib_request_Request,
 )
 from .utils import (
     ContentTooShortError,
@@ -1871,6 +1873,8 @@ class YoutubeDL(object):
 
     def urlopen(self, req):
         """ Start an HTTP download """
+        if isinstance(req, compat_basestring):
+            req = compat_urllib_request_Request(req)
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):

From e4c4bcf36f4e0bb575526701f852152f4fd976c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:12:54 +0600
Subject: [PATCH 1962/2145] [vimeo] Use compat_urllib_request_Request

---
 youtube_dl/extractor/vimeo.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 057c72f39..d1daf9672 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,7 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_request,
+    compat_urllib_request_Request,
     compat_urlparse,
 )
 from ..utils import (
@@ -47,7 +47,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'service': 'vimeo',
             'token': token,
         }))
-        login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
+        login_request = compat_urllib_request_Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
         self._set_vimeo_cookie('vuid', vuid)
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')
-        password_request = compat_urllib_request.Request(url + '/password', data)
+        password_request = compat_urllib_request_Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Referer', url)
         self._set_vimeo_cookie('vuid', vuid)
@@ -236,7 +236,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata(encode_dict({'password': password}))
         pass_url = url + '/check-password'
-        password_request = compat_urllib_request.Request(pass_url, data)
+        password_request = compat_urllib_request_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         return self._download_json(
             password_request, video_id,
@@ -265,7 +265,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
-        request = compat_urllib_request.Request(url, None, headers)
+        request = compat_urllib_request_Request(url, None, headers)
         try:
             webpage = self._download_webpage(request, video_id)
         except ExtractorError as ee:
@@ -481,7 +481,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
-        password_request = compat_urllib_request.Request(password_url, post)
+        password_request = compat_urllib_request_Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
         self._set_vimeo_cookie('vuid', vuid)
         self._set_vimeo_cookie('xsrft', token)
@@ -640,7 +640,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
 
     def _page_url(self, base_url, pagenum):
         url = '%s/page:%d/' % (base_url, pagenum)
-        request = compat_urllib_request.Request(url)
+        request = compat_urllib_request_Request(url)
         # Set the header to get a partial html page with the ids,
         # the normal page doesn't contain them.
         request.add_header('X-Requested-With', 'XMLHttpRequest')

From 67dda51722f1ce12b956782d43047b3fff390115 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 20:33:49 +0600
Subject: [PATCH 1963/2145] Rename compat_urllib_request_Request to
 sanitized_Request and move to utils

---
 youtube_dl/YoutubeDL.py       |  4 ++--
 youtube_dl/compat.py          |  8 --------
 youtube_dl/extractor/vimeo.py | 14 +++++++-------
 youtube_dl/utils.py           |  7 +++++++
 4 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b20837ce2..eedab37a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -39,7 +39,6 @@ from .compat import (
     compat_urllib_error,
     compat_urllib_request,
     compat_urllib_request_DataHandler,
-    compat_urllib_request_Request,
 )
 from .utils import (
     ContentTooShortError,
@@ -65,6 +64,7 @@ from .utils import (
     SameFileError,
     sanitize_filename,
     sanitize_path,
+    sanitized_Request,
     std_headers,
     subtitles_filename,
     UnavailableVideoError,
@@ -1874,7 +1874,7 @@ class YoutubeDL(object):
     def urlopen(self, req):
         """ Start an HTTP download """
         if isinstance(req, compat_basestring):
-            req = compat_urllib_request_Request(req)
+            req = sanitized_Request(req)
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4e3de7f51..a3e85264a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,14 +198,6 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
-
-# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
-# unwanted failures due to missing protocol
-def compat_urllib_request_Request(url, *args, **kwargs):
-    return compat_urllib_request.Request(
-        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
-
-
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d1daf9672..f392ccf1c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,6 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_request_Request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
     InAdvancePagedList,
     int_or_none,
     RegexNotFoundError,
+    sanitized_Request,
     smuggle_url,
     std_headers,
     unified_strdate,
@@ -47,7 +47,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'service': 'vimeo',
             'token': token,
         }))
-        login_request = compat_urllib_request_Request(self._LOGIN_URL, data)
+        login_request = sanitized_Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
         self._set_vimeo_cookie('vuid', vuid)
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')
-        password_request = compat_urllib_request_Request(url + '/password', data)
+        password_request = sanitized_Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Referer', url)
         self._set_vimeo_cookie('vuid', vuid)
@@ -236,7 +236,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata(encode_dict({'password': password}))
         pass_url = url + '/check-password'
-        password_request = compat_urllib_request_Request(pass_url, data)
+        password_request = sanitized_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         return self._download_json(
             password_request, video_id,
@@ -265,7 +265,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
-        request = compat_urllib_request_Request(url, None, headers)
+        request = sanitized_Request(url, None, headers)
         try:
             webpage = self._download_webpage(request, video_id)
         except ExtractorError as ee:
@@ -481,7 +481,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
-        password_request = compat_urllib_request_Request(password_url, post)
+        password_request = sanitized_Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
         self._set_vimeo_cookie('vuid', vuid)
         self._set_vimeo_cookie('xsrft', token)
@@ -640,7 +640,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
 
     def _page_url(self, base_url, pagenum):
         url = '%s/page:%d/' % (base_url, pagenum)
-        request = compat_urllib_request_Request(url)
+        request = sanitized_Request(url)
         # Set the header to get a partial html page with the ids,
         # the normal page doesn't contain them.
         request.add_header('X-Requested-With', 'XMLHttpRequest')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c0325f054..d7b737e21 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -373,6 +373,13 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def sanitized_Request(url, *args, **kwargs):
+    return compat_urllib_request.Request(
+        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From 5c2266df4b9aeb7881ed8c026a038e2a25e43734 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 22:18:17 +0600
Subject: [PATCH 1964/2145] Switch codebase to use sanitized_Request instead of
 compat_urllib_request.Request

[downloader/dash] Use sanitized_Request

[downloader/http] Use sanitized_Request

[atresplayer] Use sanitized_Request

[bambuser] Use sanitized_Request

[bliptv] Use sanitized_Request

[brightcove] Use sanitized_Request

[cbs] Use sanitized_Request

[ceskatelevize] Use sanitized_Request

[collegerama] Use sanitized_Request

[extractor/common] Use sanitized_Request

[crunchyroll] Use sanitized_Request

[dailymotion] Use sanitized_Request

[dcn] Use sanitized_Request

[dramafever] Use sanitized_Request

[dumpert] Use sanitized_Request

[eitb] Use sanitized_Request

[escapist] Use sanitized_Request

[everyonesmixtape] Use sanitized_Request

[extremetube] Use sanitized_Request

[facebook] Use sanitized_Request

[fc2] Use sanitized_Request

[flickr] Use sanitized_Request

[4tube] Use sanitized_Request

[gdcvault] Use sanitized_Request

[extractor/generic] Use sanitized_Request

[hearthisat] Use sanitized_Request

[hotnewhiphop] Use sanitized_Request

[hypem] Use sanitized_Request

[iprima] Use sanitized_Request

[ivi] Use sanitized_Request

[keezmovies] Use sanitized_Request

[letv] Use sanitized_Request

[lynda] Use sanitized_Request

[metacafe] Use sanitized_Request

[minhateca] Use sanitized_Request

[miomio] Use sanitized_Request

[meovideo] Use sanitized_Request

[mofosex] Use sanitized_Request

[moniker] Use sanitized_Request

[mooshare] Use sanitized_Request

[movieclips] Use sanitized_Request

[mtv] Use sanitized_Request

[myvideo] Use sanitized_Request

[neteasemusic] Use sanitized_Request

[nfb] Use sanitized_Request

[niconico] Use sanitized_Request

[noco] Use sanitized_Request

[nosvideo] Use sanitized_Request

[novamov] Use sanitized_Request

[nowness] Use sanitized_Request

[nuvid] Use sanitized_Request

[played] Use sanitized_Request

[pluralsight] Use sanitized_Request

[pornhub] Use sanitized_Request

[pornotube] Use sanitized_Request

[primesharetv] Use sanitized_Request

[promptfile] Use sanitized_Request

[qqmusic] Use sanitized_Request

[rtve] Use sanitized_Request

[safari] Use sanitized_Request

[sandia] Use sanitized_Request

[shared] Use sanitized_Request

[sharesix] Use sanitized_Request

[sina] Use sanitized_Request

[smotri] Use sanitized_Request

[sohu] Use sanitized_Request

[spankwire] Use sanitized_Request

[sportdeutschland] Use sanitized_Request

[streamcloud] Use sanitized_Request

[streamcz] Use sanitized_Request

[tapely] Use sanitized_Request

[tube8] Use sanitized_Request

[tubitv] Use sanitized_Request

[twitch] Use sanitized_Request

[twitter] Use sanitized_Request

[udemy] Use sanitized_Request

[vbox7] Use sanitized_Request

[veoh] Use sanitized_Request

[vessel] Use sanitized_Request

[vevo] Use sanitized_Request

[viddler] Use sanitized_Request

[videomega] Use sanitized_Request

[viewvster] Use sanitized_Request

[viki] Use sanitized_Request

[vk] Use sanitized_Request

[vodlocker] Use sanitized_Request

[voicerepublic] Use sanitized_Request

[wistia] Use sanitized_Request

[xfileshare] Use sanitized_Request

[xtube] Use sanitized_Request

[xvideos] Use sanitized_Request

[yandexmusic] Use sanitized_Request

[youku] Use sanitized_Request

[youporn] Use sanitized_Request

[youtube] Use sanitized_Request

[patreon] Use sanitized_Request

[extractor/common] Remove unused import

[nfb] PEP 8
---
 youtube_dl/YoutubeDL.py                  |  2 +-
 youtube_dl/downloader/dash.py            |  4 ++--
 youtube_dl/downloader/http.py            | 10 ++++------
 youtube_dl/extractor/atresplayer.py      |  6 +++---
 youtube_dl/extractor/bambuser.py         |  6 +++---
 youtube_dl/extractor/bliptv.py           |  8 +++-----
 youtube_dl/extractor/brightcove.py       |  6 +++---
 youtube_dl/extractor/cbs.py              |  8 +++++---
 youtube_dl/extractor/ceskatelevize.py    |  6 +++---
 youtube_dl/extractor/collegerama.py      |  4 ++--
 youtube_dl/extractor/common.py           |  4 ++--
 youtube_dl/extractor/crunchyroll.py      |  9 +++++----
 youtube_dl/extractor/dailymotion.py      |  8 +++-----
 youtube_dl/extractor/dcn.py              |  8 +++-----
 youtube_dl/extractor/dramafever.py       |  4 ++--
 youtube_dl/extractor/dumpert.py          |  8 +++++---
 youtube_dl/extractor/eitb.py             |  4 ++--
 youtube_dl/extractor/escapist.py         |  5 ++---
 youtube_dl/extractor/everyonesmixtape.py |  8 +++-----
 youtube_dl/extractor/extremetube.py      |  4 ++--
 youtube_dl/extractor/facebook.py         |  8 ++++----
 youtube_dl/extractor/fc2.py              |  5 +++--
 youtube_dl/extractor/flickr.py           |  4 ++--
 youtube_dl/extractor/fourtube.py         |  6 ++----
 youtube_dl/extractor/gdcvault.py         |  8 +++-----
 youtube_dl/extractor/generic.py          |  6 +++---
 youtube_dl/extractor/hearthisat.py       |  8 +++-----
 youtube_dl/extractor/hotnewhiphop.py     |  8 +++-----
 youtube_dl/extractor/hypem.py            | 10 ++++------
 youtube_dl/extractor/iprima.py           |  6 ++----
 youtube_dl/extractor/ivi.py              |  6 ++----
 youtube_dl/extractor/keezmovies.py       |  8 +++-----
 youtube_dl/extractor/letv.py             |  4 ++--
 youtube_dl/extractor/lynda.py            |  6 +++---
 youtube_dl/extractor/metacafe.py         |  6 +++---
 youtube_dl/extractor/minhateca.py        |  8 +++-----
 youtube_dl/extractor/miomio.py           |  4 ++--
 youtube_dl/extractor/moevideo.py         |  8 +++-----
 youtube_dl/extractor/mofosex.py          |  4 ++--
 youtube_dl/extractor/moniker.py          | 10 ++++------
 youtube_dl/extractor/mooshare.py         |  8 +++-----
 youtube_dl/extractor/movieclips.py       |  6 ++----
 youtube_dl/extractor/mtv.py              |  4 ++--
 youtube_dl/extractor/myvideo.py          |  4 ++--
 youtube_dl/extractor/neteasemusic.py     |  4 ++--
 youtube_dl/extractor/nfb.py              | 11 +++++------
 youtube_dl/extractor/niconico.py         |  6 +++---
 youtube_dl/extractor/noco.py             |  4 ++--
 youtube_dl/extractor/nosvideo.py         |  6 ++----
 youtube_dl/extractor/novamov.py          |  8 +++-----
 youtube_dl/extractor/nowness.py          | 10 +++++-----
 youtube_dl/extractor/nuvid.py            |  6 ++----
 youtube_dl/extractor/patreon.py          |  6 ++----
 youtube_dl/extractor/played.py           |  8 +++-----
 youtube_dl/extractor/pluralsight.py      |  6 +++---
 youtube_dl/extractor/pornhub.py          |  4 ++--
 youtube_dl/extractor/pornotube.py        | 10 ++++------
 youtube_dl/extractor/primesharetv.py     | 10 +++++-----
 youtube_dl/extractor/promptfile.py       |  8 +++-----
 youtube_dl/extractor/qqmusic.py          |  4 ++--
 youtube_dl/extractor/rtve.py             |  4 ++--
 youtube_dl/extractor/safari.py           |  8 +++-----
 youtube_dl/extractor/sandia.py           |  8 +++-----
 youtube_dl/extractor/shared.py           |  8 +++-----
 youtube_dl/extractor/sharesix.py         |  8 +++-----
 youtube_dl/extractor/sina.py             |  8 +++-----
 youtube_dl/extractor/smotri.py           | 10 ++++------
 youtube_dl/extractor/sohu.py             |  4 ++--
 youtube_dl/extractor/spankwire.py        |  4 ++--
 youtube_dl/extractor/sportdeutschland.py |  6 ++----
 youtube_dl/extractor/streamcloud.py      |  8 +++-----
 youtube_dl/extractor/streamcz.py         |  6 ++----
 youtube_dl/extractor/tapely.py           |  6 ++----
 youtube_dl/extractor/tube8.py            |  8 +++-----
 youtube_dl/extractor/tubitv.py           |  8 +++-----
 youtube_dl/extractor/twitch.py           |  6 +++---
 youtube_dl/extractor/twitter.py          |  4 ++--
 youtube_dl/extractor/udemy.py            |  5 +++--
 youtube_dl/extractor/vbox7.py            |  4 ++--
 youtube_dl/extractor/veoh.py             |  6 ++----
 youtube_dl/extractor/vessel.py           |  4 ++--
 youtube_dl/extractor/vevo.py             |  8 +++-----
 youtube_dl/extractor/viddler.py          |  6 ++----
 youtube_dl/extractor/videomega.py        |  4 ++--
 youtube_dl/extractor/viewster.py         |  4 ++--
 youtube_dl/extractor/viki.py             |  6 +++---
 youtube_dl/extractor/vk.py               |  4 ++--
 youtube_dl/extractor/vodlocker.py        |  8 +++-----
 youtube_dl/extractor/voicerepublic.py    |  8 +++-----
 youtube_dl/extractor/wistia.py           |  8 +++++---
 youtube_dl/extractor/xfileshare.py       |  8 +++-----
 youtube_dl/extractor/xtube.py            |  8 +++-----
 youtube_dl/extractor/xvideos.py          |  8 +++-----
 youtube_dl/extractor/yandexmusic.py      |  4 ++--
 youtube_dl/extractor/youku.py            |  9 +++++----
 youtube_dl/extractor/youporn.py          |  4 ++--
 youtube_dl/extractor/youtube.py          |  6 +++---
 97 files changed, 271 insertions(+), 353 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eedab37a7..9a8c7da05 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1189,7 +1189,7 @@ class YoutubeDL(object):
         return res
 
     def _calc_cookies(self, info_dict):
-        pr = compat_urllib_request.Request(info_dict['url'])
+        pr = sanitized_Request(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.get_header('Cookie')
 
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8b6fa2753..535f2a7fc 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import FileDownloader
-from ..compat import compat_urllib_request
+from ..utils import sanitized_Request
 
 
 class DashSegmentsFD(FileDownloader):
@@ -22,7 +22,7 @@ class DashSegmentsFD(FileDownloader):
 
         def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
-            req = compat_urllib_request.Request(target_url)
+            req = sanitized_Request(target_url)
             if remaining_bytes is not None:
                 req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index a29f5cf31..56840e026 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -7,14 +7,12 @@ import time
 import re
 
 from .common import FileDownloader
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_error,
-)
+from ..compat import compat_urllib_error
 from ..utils import (
     ContentTooShortError,
     encodeFilename,
     sanitize_open,
+    sanitized_Request,
 )
 
 
@@ -29,8 +27,8 @@ class HttpFD(FileDownloader):
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
-        basic_request = compat_urllib_request.Request(url, None, headers)
-        request = compat_urllib_request.Request(url, None, headers)
+        basic_request = sanitized_Request(url, None, headers)
+        request = sanitized_Request(url, None, headers)
 
         is_test = self.params.get('test', False)
 
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 29f8795d3..50e47ba0a 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -7,11 +7,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     int_or_none,
     float_or_none,
+    sanitized_Request,
     xpath_text,
     ExtractorError,
 )
@@ -63,7 +63,7 @@ class AtresPlayerIE(InfoExtractor):
             'j_password': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
@@ -94,7 +94,7 @@ class AtresPlayerIE(InfoExtractor):
 
         formats = []
         for fmt in ['windows', 'android_tablet']:
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
             request.add_header('User-Agent', self._USER_AGENT)
 
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 8dff1d6e3..da986e063 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -6,13 +6,13 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_str,
 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -57,7 +57,7 @@ class BambuserIE(InfoExtractor):
             'pass': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
@@ -126,7 +126,7 @@ class BambuserChannelIE(InfoExtractor):
                 '&sort=created&access_mode=0%2C1%2C2&limit={count}'
                 '&method=broadcast&format=json&vid_older_than={last}'
             ).format(user=user, count=self._STEP, last=last_id)
-            req = compat_urllib_request.Request(req_url)
+            req = sanitized_Request(req_url)
             # Without setting this header, we wouldn't get any result
             req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
             data = self._download_json(
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index c3296283d..35375f7b1 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -4,14 +4,12 @@ import re
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     clean_html,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     unescapeHTML,
     xpath_text,
     xpath_with_ns,
@@ -219,7 +217,7 @@ class BlipTVIE(InfoExtractor):
         for lang, url in subtitles_urls.items():
             # For some weird reason, blip.tv serves a video instead of subtitles
             # when we request with a common UA
-            req = compat_urllib_request.Request(url)
+            req = sanitized_Request(url)
             req.add_header('User-Agent', 'youtube-dl')
             subtitles[lang] = [{
                 # The extension is 'srt' but it's actually an 'ass' file
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 14ee05f21..f5ebae1e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -11,7 +11,6 @@ from ..compat import (
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
 )
@@ -24,6 +23,7 @@ from ..utils import (
     js_to_json,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     unescapeHTML,
     unsmuggle_url,
 )
@@ -250,7 +250,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
     def _get_video_info(self, video_id, query_str, query, referer=None):
         request_url = self._FEDERATED_URL_TEMPLATE % query_str
-        req = compat_urllib_request.Request(request_url)
+        req = sanitized_Request(request_url)
         linkBase = query.get('linkBaseURL')
         if linkBase is not None:
             referer = linkBase[0]
@@ -443,7 +443,7 @@ class BrightcoveNewIE(InfoExtractor):
                 r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
                 webpage, 'policy key', group='pk')
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
             % (account_id, video_id),
             headers={'Accept': 'application/json;pk=%s' % policy_key})
diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 43f05d278..40d07ab18 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import smuggle_url
+from ..utils import (
+    sanitized_Request,
+    smuggle_url,
+)
 
 
 class CBSIE(InfoExtractor):
@@ -48,7 +50,7 @@ class CBSIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         # Android UA is served with higher quality (720p) streams (see
         # https://github.com/rg3/youtube-dl/issues/7490)
         request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index e857e66f4..6f7b2a70d 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -5,7 +5,6 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
@@ -13,6 +12,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -100,7 +100,7 @@ class CeskaTelevizeIE(InfoExtractor):
             'requestSource': 'iVysilani',
         }
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
             data=compat_urllib_parse.urlencode(data))
 
@@ -115,7 +115,7 @@ class CeskaTelevizeIE(InfoExtractor):
         if playlist_url == 'error_region':
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
+        req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
         playlist_title = self._og_search_title(webpage)
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index fedd48490..40667a0f1 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -52,7 +52,7 @@ class CollegeRamaIE(InfoExtractor):
             }
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
             json.dumps(player_options_request))
         request.add_header('Content-Type', 'application/json')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 71bdcad5a..eb9bfa3d1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -19,7 +19,6 @@ from ..compat import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
     compat_str,
     compat_etree_fromstring,
@@ -37,6 +36,7 @@ from ..utils import (
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
+    sanitized_Request,
     unescapeHTML,
     unified_strdate,
     url_basename,
@@ -1285,7 +1285,7 @@ class InfoExtractor(object):
 
     def _get_cookies(self, url):
         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         self._downloader.cookiejar.add_cookie_header(req)
         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
 
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 6e5999c72..00d943f77 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -23,6 +23,7 @@ from ..utils import (
     int_or_none,
     lowercase_escape,
     remove_end,
+    sanitized_Request,
     unified_strdate,
     urlencode_postdata,
     xpath_text,
@@ -46,7 +47,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             'name': username,
             'password': password,
         })
-        login_request = compat_urllib_request.Request(login_url, data)
+        login_request = sanitized_Request(login_url, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
@@ -55,7 +56,7 @@ class CrunchyrollBaseIE(InfoExtractor):
 
     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
-                   else compat_urllib_request.Request(url_or_request))
+                   else sanitized_Request(url_or_request))
         # Accept-Language must be set explicitly to accept any language to avoid issues
         # similar to https://github.com/rg3/youtube-dl/issues/6797.
         # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
@@ -307,7 +308,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'video_uploader', fatal=False)
 
         playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
-        playerdata_req = compat_urllib_request.Request(playerdata_url)
+        playerdata_req = sanitized_Request(playerdata_url)
         playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
@@ -319,7 +320,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = compat_urllib_request.Request(
+            streamdata_req = sanitized_Request(
                 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
                 % (stream_id, stream_format, stream_quality),
                 compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index bc7823931..ab7f3aec4 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -7,15 +7,13 @@ import itertools
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_str,
-    compat_urllib_request,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     str_to_int,
     unescapeHTML,
 )
@@ -25,7 +23,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
     @staticmethod
     def _build_request(url):
         """Build a request with the family filter disabled"""
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 6f2fea5ff..9737cff14 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,13 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -36,7 +34,7 @@ class DCNIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
             headers={'Origin': 'http://www.dcndigital.ae'})
 
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 38e6597c8..d836c1a6c 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -16,6 +15,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -51,7 +51,7 @@ class DramaFeverBaseIE(InfoExtractor):
             'password': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index f5a31058d..e5aadcd25 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -5,8 +5,10 @@ import base64
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import qualities
+from ..utils import (
+    qualities,
+    sanitized_Request,
+)
 
 
 class DumpertIE(InfoExtractor):
@@ -32,7 +34,7 @@ class DumpertIE(InfoExtractor):
         protocol = mobj.group('protocol')
 
         url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 357a2196c..c83845fc2 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -2,11 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -57,7 +57,7 @@ class EitbIE(InfoExtractor):
 
         hls_url = media.get('HLS_SURL')
         if hls_url:
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
                 headers={'Referer': url})
             token_data = self._download_json(
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index c85b4c458..a3d7bbbcb 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,13 +3,12 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-
 from ..utils import (
     determine_ext,
     clean_html,
     int_or_none,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -75,7 +74,7 @@ class EscapistIE(InfoExtractor):
         video_id = ims_video['videoID']
         key = ims_video['hash']
 
-        config_req = compat_urllib_request.Request(
+        config_req = sanitized_Request(
             'http://www.escapistmagazine.com/videos/'
             'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
         config_req.add_header('Referer', url)
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index d872d828f..493d38af8 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -42,7 +40,7 @@ class EveryonesMixtapeIE(InfoExtractor):
         playlist_id = mobj.group('id')
 
         pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
-        pllist_req = compat_urllib_request.Request(pllist_url)
+        pllist_req = sanitized_Request(pllist_url)
         pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
 
         playlist_list = self._download_json(
@@ -55,7 +53,7 @@ class EveryonesMixtapeIE(InfoExtractor):
             raise ExtractorError('Playlist id not found')
 
         pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
-        pl_req = compat_urllib_request.Request(pl_url)
+        pl_req = sanitized_Request(pl_url)
         pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
         playlist = self._download_json(
             pl_req, playlist_id, note='Downloading playlist info')
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index c5677c82b..3403581fd 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     int_or_none,
+    sanitized_Request,
     str_to_int,
 )
 
@@ -37,7 +37,7 @@ class ExtremeTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f53c51615..fd854411b 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -10,11 +10,11 @@ from ..compat import (
     compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
     limit_length,
+    sanitized_Request,
     urlencode_postdata,
     get_element_by_id,
     clean_html,
@@ -73,7 +73,7 @@ class FacebookIE(InfoExtractor):
         if useremail is None:
             return
 
-        login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
+        login_page_req = sanitized_Request(self._LOGIN_URL)
         login_page_req.add_header('Cookie', 'locale=en_US')
         login_page = self._download_webpage(login_page_req, None,
                                             note='Downloading login page',
@@ -94,7 +94,7 @@ class FacebookIE(InfoExtractor):
             'timezone': '-60',
             'trynum': '1',
         }
-        request = compat_urllib_request.Request(self._LOGIN_URL, urlencode_postdata(login_form))
+        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         try:
             login_results = self._download_webpage(request, None,
@@ -109,7 +109,7 @@ class FacebookIE(InfoExtractor):
                     r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
                 'name_action_selected': 'dont_save',
             }
-            check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
+            check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
             check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             check_response = self._download_webpage(check_req, None,
                                                     note='Confirming login')
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index a406945e8..92e8c571f 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -12,6 +12,7 @@ from ..compat import (
 from ..utils import (
     encode_dict,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -57,7 +58,7 @@ class FC2IE(InfoExtractor):
         }
 
         login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@@ -66,7 +67,7 @@ class FC2IE(InfoExtractor):
             return False
 
         # this is also needed
-        login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+        login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
         self._download_webpage(
             login_redir, None, note='Login redirect', errnote='Login redirect failed')
 
diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 2fe76d661..91cd46e76 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     find_xpath_attr,
+    sanitized_Request,
 )
 
 
@@ -30,7 +30,7 @@ class FlickrIE(InfoExtractor):
         video_id = mobj.group('id')
         video_uploader_id = mobj.group('uploader_id')
         webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
-        req = compat_urllib_request.Request(webpage_url)
+        req = sanitized_Request(webpage_url)
         req.add_header(
             'User-Agent',
             # it needs a more recent version
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index fb6d108c0..fc4a5a0fb 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     parse_duration,
     parse_iso8601,
+    sanitized_Request,
     str_to_int,
 )
 
@@ -93,7 +91,7 @@ class FourTubeIE(InfoExtractor):
             b'Content-Type': b'application/x-www-form-urlencoded',
             b'Origin': b'http://www.4tube.com',
         }
-        token_req = compat_urllib_request.Request(token_url, b'{}', headers)
+        token_req = sanitized_Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)
         formats = [{
             'url': tokens[format]['token'],
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index a6834db43..3befd3e7b 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     remove_end,
     HEADRequest,
+    sanitized_Request,
 )
 
 
@@ -125,7 +123,7 @@ class GDCVaultIE(InfoExtractor):
             'password': password,
         }
 
-        request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form))
+        request = sanitized_Request(login_url, compat_urllib_parse.urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(request, display_id, 'Logging in')
         start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2b934148d..5075d131e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,7 +11,6 @@ from .youtube import YoutubeIE
 from ..compat import (
     compat_etree_fromstring,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
 )
@@ -22,6 +21,7 @@ from ..utils import (
     HEADRequest,
     is_html,
     orderedSet,
+    sanitized_Request,
     smuggle_url,
     unescapeHTML,
     unified_strdate,
@@ -1215,7 +1215,7 @@ class GenericIE(InfoExtractor):
 
         full_response = None
         if head_response is False:
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.add_header('Accept-Encoding', '*')
             full_response = self._request_webpage(request, video_id)
             head_response = full_response
@@ -1244,7 +1244,7 @@ class GenericIE(InfoExtractor):
                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
         if not full_response:
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
             # making it impossible to download only chunk of the file (yet we need only 512kB to
             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index a19b31ac0..7d8698655 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     HEADRequest,
+    sanitized_Request,
     str_to_int,
     urlencode_postdata,
     urlhandle_detect_ext,
@@ -47,7 +45,7 @@ class HearThisAtIE(InfoExtractor):
             r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
 
         payload = urlencode_postdata({'tracks[]': track_id})
-        req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
+        req = sanitized_Request(self._PLAYLIST_URL, payload)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
         track = self._download_json(req, track_id, 'Downloading playlist')[0]
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 651784b73..31e219945 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     HEADRequest,
+    sanitized_Request,
 )
 
 
@@ -41,7 +39,7 @@ class HotNewHipHopIE(InfoExtractor):
             ('mediaType', 's'),
             ('mediaId', video_id),
         ])
-        r = compat_urllib_request.Request(
+        r = sanitized_Request(
             'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
         r.add_header('Content-Type', 'application/x-www-form-urlencoded')
         mkd = self._download_json(
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index aa0724a02..cca3dd498 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -4,12 +4,10 @@ import json
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -32,7 +30,7 @@ class HypemIE(InfoExtractor):
         data = {'ax': 1, 'ts': time.time()}
         data_encoded = compat_urllib_parse.urlencode(data)
         complete_url = url + "?" + data_encoded
-        request = compat_urllib_request.Request(complete_url)
+        request = sanitized_Request(complete_url)
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
         cookie = urlh.headers.get('Set-Cookie', '')
@@ -52,7 +50,7 @@ class HypemIE(InfoExtractor):
         title = track['song']
 
         serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             serve_url, '', {'Content-Type': 'application/json'})
         request.add_header('cookie', cookie)
         song_data = self._download_json(request, track_id, 'Downloading metadata')
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 821c8ec10..36baf3245 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -6,12 +6,10 @@ from random import random
 from math import floor
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
     remove_end,
+    sanitized_Request,
 )
 
 
@@ -61,7 +59,7 @@ class IPrimaIE(InfoExtractor):
             (floor(random() * 1073741824), floor(random() * 1073741824))
         )
 
-        req = compat_urllib_request.Request(player_url)
+        req = sanitized_Request(player_url)
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index e82594444..029878d24 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -5,11 +5,9 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -78,7 +76,7 @@ class IviIE(InfoExtractor):
             ]
         }
 
-        request = compat_urllib_request.Request(api_url, json.dumps(data))
+        request = sanitized_Request(api_url, json.dumps(data))
 
         video_json_page = self._download_webpage(
             request, video_id, 'Downloading video JSON')
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 82eddec51..d79261bb5 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -4,10 +4,8 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import sanitized_Request
 
 
 class KeezMoviesIE(InfoExtractor):
@@ -26,7 +24,7 @@ class KeezMoviesIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index effd9eb92..be648000e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -8,13 +8,13 @@ import time
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_ord,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
+    sanitized_Request,
     int_or_none,
     encode_data_uri,
 )
@@ -114,7 +114,7 @@ class LetvIE(InfoExtractor):
             'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.letv.com'
         }
-        play_json_req = compat_urllib_request.Request(
+        play_json_req = sanitized_Request(
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
         cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 3d7e7e003..d4e1ae99d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -7,12 +7,12 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -35,7 +35,7 @@ class LyndaBaseIE(InfoExtractor):
             'remember': 'false',
             'stayPut': 'false'
         }
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
@@ -64,7 +64,7 @@ class LyndaBaseIE(InfoExtractor):
                     'remember': 'false',
                     'stayPut': 'false',
                 }
-                request = compat_urllib_request.Request(
+                request = sanitized_Request(
                     self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
                 login_page = self._download_webpage(
                     request, None,
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 6e2e73a51..3c786a36d 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -7,12 +7,12 @@ from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -117,7 +117,7 @@ class MetacafeIE(InfoExtractor):
             'filters': '0',
             'submit': "Continue - I'm over 18",
         }
-        request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
+        request = sanitized_Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self.report_age_confirmation()
         self._download_webpage(request, None, False, 'Unable to confirm age')
@@ -142,7 +142,7 @@ class MetacafeIE(InfoExtractor):
                 return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
 
         # Retrieve video webpage to extract further information
-        req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
+        req = sanitized_Request('http://www.metacafe.com/watch/%s/' % video_id)
 
         # AnyClip videos require the flashversion cookie so that we get the link
         # to the mp4 file
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index 14934b7ec..e46b23a6f 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -2,14 +2,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_filesize,
+    sanitized_Request,
 )
 
 
@@ -39,7 +37,7 @@ class MinhatecaIE(InfoExtractor):
             ('fileId', video_id),
             ('__RequestVerificationToken', token),
         ]
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://minhateca.com.br/action/License/Download',
             data=compat_urllib_parse.urlencode(token_data))
         req.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index ce391c759..170ebd9eb 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     xpath_text,
     int_or_none,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -63,7 +63,7 @@ class MioMioIE(InfoExtractor):
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
             video_id)
 
-        vid_config_request = compat_urllib_request.Request(
+        vid_config_request = sanitized_Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
             headers=http_headers)
 
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 5a66302f6..d930b9634 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,13 +5,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -80,7 +78,7 @@ class MoeVideoIE(InfoExtractor):
         ]
         r_json = json.dumps(r)
         post = compat_urllib_parse.urlencode({'r': r_json})
-        req = compat_urllib_request.Request(self._API_URL, post)
+        req = sanitized_Request(self._API_URL, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
         response = self._download_json(req, video_id)
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index 9bf99a54a..f8226cbb2 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
 )
+from ..utils import sanitized_Request
 
 
 class MofosexIE(InfoExtractor):
@@ -29,7 +29,7 @@ class MofosexIE(InfoExtractor):
         video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 7c0c4e50e..f6bf94f2f 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,13 +5,11 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     remove_start,
+    sanitized_Request,
 )
 
 
@@ -81,7 +79,7 @@ class MonikerIE(InfoExtractor):
             orig_webpage, 'builtin URL', default=None, group='url')
 
         if builtin_url:
-            req = compat_urllib_request.Request(builtin_url)
+            req = sanitized_Request(builtin_url)
             req.add_header('Referer', url)
             webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
             title = self._og_search_title(orig_webpage).strip()
@@ -94,7 +92,7 @@ class MonikerIE(InfoExtractor):
             headers = {
                 b'Content-Type': b'application/x-www-form-urlencoded',
             }
-            req = compat_urllib_request.Request(url, post, headers)
+            req = sanitized_Request(url, post, headers)
             webpage = self._download_webpage(
                 req, video_id, note='Downloading video page ...')
 
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 7603af5e2..7cc7f054f 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -59,7 +57,7 @@ class MooshareIE(InfoExtractor):
             'hash': hash_key,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index b8c43a163..1564cb71f 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
+from ..utils import sanitized_Request
 
 
 class MovieClipsIE(InfoExtractor):
@@ -25,7 +23,7 @@ class MovieClipsIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         # it doesn't work if it thinks the browser it's too old
         req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
         webpage = self._download_webpage(req, display_id)
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 302c9bf35..d887583e6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_str,
 )
 from ..utils import (
@@ -13,6 +12,7 @@ from ..utils import (
     find_xpath_attr,
     fix_xml_ampersands,
     HEADRequest,
+    sanitized_Request,
     unescapeHTML,
     url_basename,
     RegexNotFoundError,
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
     def _extract_mobile_video_formats(self, mtvn_id):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
-        req = compat_urllib_request.Request(webpage_url)
+        req = sanitized_Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
         req.add_header('User-Agent', 'curl/7')
         webpage = self._download_webpage(req, mtvn_id,
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index c96f472a3..36ab388b2 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -11,10 +11,10 @@ from ..compat import (
     compat_ord,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -83,7 +83,7 @@ class MyVideoIE(InfoExtractor):
 
         mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
         if mobj is not None:
-            request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
+            request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
             response = self._download_webpage(request, video_id,
                                               'Downloading video info')
             info = json.loads(base64.b64decode(response).decode('utf-8'))
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index bb3362069..15eca825a 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -8,11 +8,11 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_request,
     compat_urllib_parse,
     compat_str,
     compat_itertools_count,
 )
+from ..utils import sanitized_Request
 
 
 class NetEaseMusicBaseIE(InfoExtractor):
@@ -56,7 +56,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
         return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
-        req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
         req.add_header('Referer', self._API_BASE)
         return self._download_json(req, video_id, note)
 
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index ea077254b..5bd15f7a7 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class NFBIE(InfoExtractor):
@@ -40,8 +38,9 @@ class NFBIE(InfoExtractor):
         uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
                                            page, 'director name', fatal=False)
 
-        request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
-                                                compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
+        request = sanitized_Request(
+            'https://www.nfb.ca/film/%s/player_config' % video_id,
+            compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
 
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index bda1cff05..586e52a4a 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -8,7 +8,6 @@ import datetime
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    sanitized_Request,
     xpath_text,
     determine_ext,
 )
@@ -102,7 +102,7 @@ class NiconicoIE(InfoExtractor):
             'password': password,
         }
         login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(
             request, None, note='Logging in', errnote='Unable to log in')
@@ -145,7 +145,7 @@ class NiconicoIE(InfoExtractor):
                 'k': thumb_play_key,
                 'v': video_id
             })
-            flv_info_request = compat_urllib_request.Request(
+            flv_info_request = sanitized_Request(
                 'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
                 {'Content-Type': 'application/x-www-form-urlencoded'})
             flv_info_webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index a53e27b27..76bd21e6d 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -9,7 +9,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     clean_html,
@@ -17,6 +16,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -74,7 +74,7 @@ class NocoIE(InfoExtractor):
             'username': username,
             'password': password,
         }
-        request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
 
         login = self._download_json(request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/nosvideo.py b/youtube_dl/extractor/nosvideo.py
index f5ef856db..eab816e49 100644
--- a/youtube_dl/extractor/nosvideo.py
+++ b/youtube_dl/extractor/nosvideo.py
@@ -4,11 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
     urlencode_postdata,
     xpath_text,
     xpath_with_ns,
@@ -41,7 +39,7 @@ class NosVideoIE(InfoExtractor):
             'op': 'download1',
             'method_free': 'Continue to Video',
         }
-        req = compat_urllib_request.Request(url, urlencode_postdata(fields))
+        req = sanitized_Request(url, urlencode_postdata(fields))
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(req, video_id,
                                          'Downloading download page')
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 6b15fc2e5..6163e8855 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     NO_DEFAULT,
     encode_dict,
+    sanitized_Request,
     urlencode_postdata,
 )
 
@@ -65,7 +63,7 @@ class NovaMovIE(InfoExtractor):
                 'post url', default=url, group='url')
             if not post_url.startswith('http'):
                 post_url = compat_urlparse.urljoin(url, post_url)
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 post_url, urlencode_postdata(encode_dict(fields)))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             request.add_header('Referer', post_url)
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 0fba55833..d480fb58c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 
 from .brightcove import BrightcoveLegacyIE
 from .common import InfoExtractor
-from ..utils import ExtractorError
-from ..compat import (
-    compat_str,
-    compat_urllib_request,
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -37,7 +37,7 @@ class NownessBaseIE(InfoExtractor):
 
     def _api_request(self, url, request_path):
         display_id = self._match_id(url)
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://api.nowness.com/api/' + request_path % display_id,
             headers={
                 'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 57928f2ae..9fa7cefad 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     parse_duration,
+    sanitized_Request,
     unified_strdate,
 )
 
@@ -33,7 +31,7 @@ class NuvidIE(InfoExtractor):
         formats = []
 
         for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 'http://m.nuvid.com/play/%s' % video_id)
             request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
             webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 6cdc2638b..ec8876c28 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-)
+from ..utils import js_to_json
 
 
 class PatreonIE(InfoExtractor):
@@ -65,7 +63,7 @@ class PatreonIE(InfoExtractor):
             'password': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://www.patreon.com/processLogin',
             compat_urllib_parse.urlencode(login_form).encode('utf-8')
         )
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 8a1c296dd..2856af96f 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,12 +5,10 @@ import re
 import os.path
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -46,7 +44,7 @@ class PlayedIE(InfoExtractor):
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }
-        req = compat_urllib_request.Request(url, post, headers)
+        req = sanitized_Request(url, post, headers)
         webpage = self._download_webpage(
             req, video_id, note='Downloading video page ...')
 
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 792316db8..aa7dbcb63 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -8,13 +8,13 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    sanitized_Request,
 )
 
 
@@ -73,7 +73,7 @@ class PluralsightIE(PluralsightBaseIE):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
@@ -181,7 +181,7 @@ class PluralsightIE(PluralsightBaseIE):
                     'mt': ext,
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
-                request = compat_urllib_request.Request(
+                request = sanitized_Request(
                     '%s/training/Player/ViewClip' % self._API_BASE,
                     json.dumps(clip_post).encode('utf-8'))
                 request.add_header('Content-Type', 'application/json;charset=utf-8')
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index a656ad85a..965940a4b 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -8,10 +8,10 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
     str_to_int,
 )
 from ..aes import (
@@ -53,7 +53,7 @@ class PornHubIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 34735c51e..5398e708b 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -46,7 +44,7 @@ class PornotubeIE(InfoExtractor):
             'authenticationSpaceKey': originAuthenticationSpaceKey,
             'credentials': 'Clip Application',
         }
-        token_req = compat_urllib_request.Request(
+        token_req = sanitized_Request(
             'https://api.aebn.net/auth/v1/token/primal',
             data=json.dumps(token_req_data).encode('utf-8'))
         token_req.add_header('Content-Type', 'application/json')
@@ -56,7 +54,7 @@ class PornotubeIE(InfoExtractor):
         token = token_answer['tokenKey']
 
         # Get video URL
-        delivery_req = compat_urllib_request.Request(
+        delivery_req = sanitized_Request(
             'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
         delivery_req.add_header('Authorization', token)
         delivery_info = self._download_json(
@@ -64,7 +62,7 @@ class PornotubeIE(InfoExtractor):
         video_url = delivery_info['mediaUrl']
 
         # Get additional info (title etc.)
-        info_req = compat_urllib_request.Request(
+        info_req = sanitized_Request(
             'https://api.aebn.net/content/v1/clips/%s?expand='
             'title,description,primaryImageNumber,startSecond,endSecond,'
             'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 304359dc5..85aae9576 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
+from ..compat import compat_urllib_parse
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
 )
-from ..utils import ExtractorError
 
 
 class PrimeShareTVIE(InfoExtractor):
@@ -41,7 +41,7 @@ class PrimeShareTVIE(InfoExtractor):
             webpage, 'wait time', default=7)) + 1
         self._sleep(wait_time, video_id)
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             url, compat_urllib_parse.urlencode(fields), headers)
         video_page = self._download_webpage(
             req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 8190ed676..d5357283a 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,13 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     determine_ext,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -37,7 +35,7 @@ class PromptFileIE(InfoExtractor):
 
         fields = self._hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
+        req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(
             req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c98539f6a..1ba3bbddf 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -7,11 +7,11 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    sanitized_Request,
     strip_jsonp,
     unescapeHTML,
     clean_html,
 )
-from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -201,7 +201,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
         singer_desc = None
 
         if singer_id:
-            req = compat_urllib_request.Request(
+            req = sanitized_Request(
                 'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
             req.add_header(
                 'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 0fe6356db..603d7bd00 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,11 +6,11 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     float_or_none,
     remove_end,
+    sanitized_Request,
     std_headers,
     struct_unpack,
 )
@@ -102,7 +102,7 @@ class RTVEALaCartaIE(InfoExtractor):
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
-        png_request = compat_urllib_request.Request(png_url)
+        png_request = sanitized_Request(png_url)
         png_request.add_header('Referer', url)
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index e9e33d0a3..919704261 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -6,12 +6,10 @@ import re
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
 
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
     smuggle_url,
     std_headers,
 )
@@ -58,7 +56,7 @@ class SafariBaseIE(InfoExtractor):
             'next': '',
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
index 9c88167f0..759898a49 100644
--- a/youtube_dl/extractor/sandia.py
+++ b/youtube_dl/extractor/sandia.py
@@ -6,14 +6,12 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
     mimetype2ext,
+    sanitized_Request,
     unified_strdate,
 )
 
@@ -37,7 +35,7 @@ class SandiaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index c5636e8e9..8eda3c864 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -46,7 +44,7 @@ class SharedIE(InfoExtractor):
                 'Video %s does not exist' % video_id, expected=True)
 
         download_form = self._hidden_inputs(webpage)
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index ac3e3adf2..f1ea9bdb2 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     parse_duration,
+    sanitized_Request,
 )
 
 
@@ -50,7 +48,7 @@ class ShareSixIE(InfoExtractor):
             'method_free': 'Free'
         }
         post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
+        req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
         webpage = self._download_webpage(req, video_id,
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index 0891a441f..b2258a0f6 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,10 +4,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class SinaIE(InfoExtractor):
@@ -61,7 +59,7 @@ class SinaIE(InfoExtractor):
         if mobj.group('token') is not None:
             # The video id is in the redirected url
             self.to_screen('Getting video id')
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.get_method = lambda: 'HEAD'
             (_, urlh) = self._download_webpage_handle(request, 'NA', False)
             return self._real_extract(urlh.geturl())
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 35a81ee87..30210c8a3 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,13 +7,11 @@ import hashlib
 import uuid
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
     unified_strdate,
 )
 
@@ -176,7 +174,7 @@ class SmotriIE(InfoExtractor):
         if video_password:
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
@@ -339,7 +337,7 @@ class SmotriBroadcastIE(InfoExtractor):
                 'password': password,
             }
 
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             broadcast_page = self._download_webpage(
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ba2d5e19b..daf6ad555 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,11 +6,11 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_request,
     compat_urllib_parse,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -96,7 +96,7 @@ class SohuIE(InfoExtractor):
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
 
-            req = compat_urllib_request.Request(base_data_url + vid_id)
+            req = sanitized_Request(base_data_url + vid_id)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 9e8fb35b2..692fd78e8 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
 )
 from ..utils import (
+    sanitized_Request,
     str_to_int,
     unified_strdate,
 )
@@ -51,7 +51,7 @@ class SpankwireIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        req = compat_urllib_request.Request('http://www.' + mobj.group('url'))
+        req = sanitized_Request('http://www.' + mobj.group('url'))
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 7ec6c613f..ebb75f059 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -4,11 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -54,7 +52,7 @@ class SportDeutschlandIE(InfoExtractor):
 
         api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
             sport_id, video_id)
-        req = compat_urllib_request.Request(api_url, headers={
+        req = sanitized_Request(api_url, headers={
             'Accept': 'application/vnd.vidibus.v2.html+json',
             'Referer': url,
         })
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index d4e134015..77841b946 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -4,10 +4,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class StreamcloudIE(InfoExtractor):
@@ -43,7 +41,7 @@ class StreamcloudIE(InfoExtractor):
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }
-        req = compat_urllib_request.Request(url, post, headers)
+        req = sanitized_Request(url, post, headers)
 
         webpage = self._download_webpage(
             req, video_id, note='Downloading video page ...')
diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index e92b93285..d3d2b7eb7 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -5,11 +5,9 @@ import hashlib
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -54,7 +52,7 @@ class StreamCZIE(InfoExtractor):
         video_id = self._match_id(url)
         api_path = '/episode/%s' % video_id
 
-        req = compat_urllib_request.Request(self._API_URL + api_path)
+        req = sanitized_Request(self._API_URL + api_path)
         req.add_header('Api-Password', _get_api_key(api_path))
         data = self._download_json(req, video_id)
 
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 744f9db38..ed560bd24 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -4,14 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     clean_html,
     ExtractorError,
     float_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -53,7 +51,7 @@ class TapelyIE(InfoExtractor):
         display_id = mobj.group('id')
 
         playlist_url = self._API_URL.format(display_id)
-        request = compat_urllib_request.Request(playlist_url)
+        request = sanitized_Request(playlist_url)
         request.add_header('X-Requested-With', 'XMLHttpRequest')
         request.add_header('Accept', 'application/json')
         request.add_header('Referer', url)
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index c9cb69333..46ef61ff5 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -4,12 +4,10 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     int_or_none,
+    sanitized_Request,
     str_to_int,
 )
 from ..aes import aes_decrypt_text
@@ -42,7 +40,7 @@ class Tube8IE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, display_id)
 
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 4f86b3ee9..6d78b5dfe 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -5,13 +5,11 @@ import codecs
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -44,7 +42,7 @@ class TubiTvIE(InfoExtractor):
             'password': password,
         }
         payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
-        request = compat_urllib_request.Request(self._LOGIN_URL, payload)
+        request = sanitized_Request(self._LOGIN_URL, payload)
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_page = self._download_webpage(
             request, None, False, 'Wrong login info')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 3ec08b674..69882da63 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -11,7 +11,6 @@ from ..compat import (
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -20,6 +19,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -48,7 +48,7 @@ class TwitchBaseIE(InfoExtractor):
         for cookie in self._downloader.cookiejar:
             if cookie.name == 'api_token':
                 headers['Twitch-Api-Token'] = cookie.value
-        request = compat_urllib_request.Request(url, headers=headers)
+        request = sanitized_Request(url, headers=headers)
         response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
         self._handle_error(response)
         return response
@@ -80,7 +80,7 @@ class TwitchBaseIE(InfoExtractor):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
         request.add_header('Referer', redirect_url)
         response = self._download_webpage(
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 055047340..a161f046b 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -4,13 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     xpath_text,
     remove_end,
     int_or_none,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -81,7 +81,7 @@ class TwitterCardIE(InfoExtractor):
         config = None
         formats = []
         for user_agent in USER_AGENTS:
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 365d8b4bf..825172806 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -58,7 +59,7 @@ class UdemyIE(InfoExtractor):
             for header, value in headers.items():
                 url_or_request.add_header(header, value)
         else:
-            url_or_request = compat_urllib_request.Request(url_or_request, headers=headers)
+            url_or_request = sanitized_Request(url_or_request, headers=headers)
 
         response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
         self._handle_error(response)
@@ -89,7 +90,7 @@ class UdemyIE(InfoExtractor):
             'password': password.encode('utf-8'),
         })
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._ORIGIN_URL)
         request.add_header('Origin', self._ORIGIN_URL)
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 722eb5236..1e740fbe6 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -49,7 +49,7 @@ class Vbox7IE(InfoExtractor):
 
         info_url = "http://vbox7.com/play/magare.do"
         data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
-        info_request = compat_urllib_request.Request(info_url, data)
+        info_request = sanitized_Request(info_url, data)
         info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
         if info_response is None:
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 01e258e32..9633f7ffe 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -4,12 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     int_or_none,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -110,7 +108,7 @@ class VeohIE(InfoExtractor):
         if 'class="adultwarning-container"' in webpage:
             self.report_age_confirmation()
             age_limit = 18
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.add_header('Cookie', 'confirmedAdult=true')
             webpage = self._download_webpage(request, video_id)
 
diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 3c8d2a943..1a0ff3395 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -4,10 +4,10 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -33,7 +33,7 @@ class VesselIE(InfoExtractor):
     @staticmethod
     def make_json_request(url, data):
         payload = json.dumps(data).encode('utf-8')
-        req = compat_urllib_request.Request(url, payload)
+        req = sanitized_Request(url, payload)
         req.add_header('Content-Type', 'application/json; charset=utf-8')
         return req
 
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 4c0de354f..571289421 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_urllib_request,
-)
+from ..compat import compat_etree_fromstring
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -73,7 +71,7 @@ class VevoIE(InfoExtractor):
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
     def _real_initialize(self):
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://www.vevo.com/auth', data=b'')
         webpage = self._download_webpage(
             req, None,
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 8516a2940..40ffbad2a 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -4,9 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
-)
-from ..compat import (
-    compat_urllib_request
+    sanitized_Request,
 )
 
 
@@ -65,7 +63,7 @@ class ViddlerIE(InfoExtractor):
             'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
             video_id)
         headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
-        request = compat_urllib_request.Request(json_url, None, headers)
+        request = sanitized_Request(json_url, None, headers)
         data = self._download_json(request, video_id)['video']
 
         formats = []
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 78ff6310a..87aca327b 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..utils import sanitized_Request
 
 
 class VideoMegaIE(InfoExtractor):
@@ -30,7 +30,7 @@ class VideoMegaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
-        req = compat_urllib_request.Request(iframe_url)
+        req = sanitized_Request(iframe_url)
         req.add_header('Referer', url)
         req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 7cf930d69..185b1c119 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
 )
@@ -13,6 +12,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     HEADRequest,
 )
 
@@ -76,7 +76,7 @@ class ViewsterIE(InfoExtractor):
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         request.add_header('Accept', self._ACCEPT_HEADER)
         request.add_header('Auth-token', self._AUTH_TOKEN)
         return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index ddbd395c8..a63c23617 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -7,14 +7,14 @@ import hmac
 import hashlib
 import itertools
 
+from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_iso8601,
+    sanitized_Request,
 )
-from ..compat import compat_urllib_request
-from .common import InfoExtractor
 
 
 class VikiBaseIE(InfoExtractor):
@@ -43,7 +43,7 @@ class VikiBaseIE(InfoExtractor):
             hashlib.sha1
         ).hexdigest()
         url = self._API_URL_TEMPLATE % (query, sig)
-        return compat_urllib_request.Request(
+        return sanitized_Request(
             url, json.dumps(post_data).encode('utf-8')) if post_data else url
 
     def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 01960b827..d99a42a9f 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -8,11 +8,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
     orderedSet,
+    sanitized_Request,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -182,7 +182,7 @@ class VKIE(InfoExtractor):
             'pass': password.encode('cp1251'),
         })
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://login.vk.com/?act=login',
             compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index ccf1928b5..be0a2780f 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,10 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class VodlockerIE(InfoExtractor):
@@ -31,7 +29,7 @@ class VodlockerIE(InfoExtractor):
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!
             post = compat_urllib_parse.urlencode(fields)
-            req = compat_urllib_request.Request(url, post)
+            req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
             webpage = self._download_webpage(
                 req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 254383d6c..93d15a556 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -37,7 +35,7 @@ class VoiceRepublicIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             compat_urlparse.urljoin(url, '/talks/%s' % display_id))
         # Older versions of Firefox get redirected to an "upgrade browser" page
         req.add_header('User-Agent', 'youtube-dl')
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 13a079151..fdb16d91c 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
+)
 
 
 class WistiaIE(InfoExtractor):
@@ -23,7 +25,7 @@ class WistiaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(self._API_URL.format(video_id))
+        request = sanitized_Request(self._API_URL.format(video_id))
         request.add_header('Referer', url)  # Some videos require this.
         data_json = self._download_json(request, video_id)
         if data_json.get('error'):
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 952515c98..a3236e66c 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -4,14 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     encode_dict,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -106,7 +104,7 @@ class XFileShareIE(InfoExtractor):
 
             post = compat_urllib_parse.urlencode(encode_dict(fields))
 
-            req = compat_urllib_request.Request(url, post)
+            req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 779e4f46a..a1fe24050 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse_unquote,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     parse_duration,
+    sanitized_Request,
     str_to_int,
 )
 
@@ -32,7 +30,7 @@ class XTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 5dcf2fdd1..710ad5041 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     clean_html,
     ExtractorError,
     determine_ext,
+    sanitized_Request,
 )
 
 
@@ -48,7 +46,7 @@ class XVideosIE(InfoExtractor):
             'url': video_url,
         }]
 
-        android_req = compat_urllib_request.Request(url)
+        android_req = sanitized_Request(url)
         android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
         android_webpage = self._download_webpage(android_req, video_id, fatal=False)
 
diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 08dc81f3a..d3cc1a29f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -8,11 +8,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     int_or_none,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -154,7 +154,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
         if len(tracks) < len(track_ids):
             present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
             missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 'https://music.yandex.ru/handlers/track-entries.jsx',
                 compat_urllib_parse.urlencode({
                     'entries': ','.join(missing_track_ids),
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 2e81d9223..69ecc837a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -4,12 +4,13 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
-
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
-    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -187,7 +188,7 @@ class YoukuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         def retrieve_data(req_url, note):
-            req = compat_urllib_request.Request(req_url)
+            req = sanitized_Request(req_url)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 9bf8d1eeb..dd724085a 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     int_or_none,
+    sanitized_Request,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -63,7 +63,7 @@ class YouPornIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         request.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(request, display_id)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0246050c2..cfe9eed55 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -20,7 +20,6 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
     compat_str,
 )
@@ -35,6 +34,7 @@ from ..utils import (
     orderedSet,
     parse_duration,
     remove_start,
+    sanitized_Request,
     smuggle_url,
     str_to_int,
     unescapeHTML,
@@ -114,7 +114,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
 
-        req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
+        req = sanitized_Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
             req, None,
             note='Logging in', errnote='unable to log in', fatal=False)
@@ -147,7 +147,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
             tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
 
-            tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
+            tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(
                 tfa_req, None,
                 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)

From 20e98bf6c0df1dc10651d5b835d42385db9bbdc7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Nov 2015 18:07:58 +0100
Subject: [PATCH 1965/2145] release 2015.11.23

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2baf1ac42..3e2c0b4ac 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.21'
+__version__ = '2015.11.23'

From 032f2f260f7ef7a04d1499b26b97c1643fe993b0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 03:38:46 +0100
Subject: [PATCH 1966/2145] README: Document which other programs may be
 helpful (Fixes #7621)

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index b286651cd..957059f8c 100644
--- a/README.md
+++ b/README.md
@@ -534,6 +534,12 @@ Most people asking this question are not aware that youtube-dl now defaults to d
 
 Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
 
+### Do I need any other programs?
+
+youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
+
+Some videos or video formats can also be only downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed.
+
 ### I have downloaded a video but how can I play it?
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).

From 4c7d816dd70a7b10dc096d815da55d41c46f9b59 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 07:45:02 +0100
Subject: [PATCH 1967/2145] [jsinterp] Adapt to updated YouTube code generation
 (Fixes #7623, fixes #7624, fixes #7625, fixes #7626)

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 9bc855144..2191e8b89 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,7 +214,7 @@ class JSInterpreter(object):
         obj = {}
         obj_m = re.search(
             (r'(?:var\s+)?%s\s*=\s*\{' % re.escape(objname)) +
-            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\})*)' +
+            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
             r'\}\s*;',
             self.code)
         fields = obj_m.group('fields')

From ba7a92b0ced8cce0c800e287777f1654b728f17c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 07:46:38 +0100
Subject: [PATCH 1968/2145] release 2015.11.24

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e2c0b4ac..5ecff39fb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.23'
+__version__ = '2015.11.24'

From 7689413e42334ff79e671a4c90869077e9f9835b Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 24 Nov 2015 23:06:21 +0600
Subject: [PATCH 1969/2145] [README.md] Mention mplayer and mpv in "other
 programs" question

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 957059f8c..b85e08e78 100644
--- a/README.md
+++ b/README.md
@@ -538,7 +538,7 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
-Some videos or video formats can also be only downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed.
+Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
 ### I have downloaded a video but how can I play it?
 

From 527ca1da4f778c5ad13ff3c74a8d311adb9aaff3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 26 Nov 2015 21:24:10 +0100
Subject: [PATCH 1970/2145] [audimedia] Add new extractor(closes #7654)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/audimedia.py | 80 +++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 youtube_dl/extractor/audimedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..f17c22691 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -38,6 +38,7 @@ from .arte import (
 )
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
+from .audimedia import AudiMediaIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
new file mode 100644
index 000000000..cad503510
--- /dev/null
+++ b/youtube_dl/extractor/audimedia.py
@@ -0,0 +1,80 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    sanitized_Request,
+)
+
+
+class AudiMediaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audimedia\.tv/(?:en|de)/vid/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://audimedia.tv/en/vid/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test',
+        'md5': '79a8b71c46d49042609795ab59779b66',
+        'info_dict': {
+            'id': '1564',
+            'ext': 'mp4',
+            'title': '60 Seconds of Audi Sport 104/2015 - WEC Bahrain, Rookie Test',
+            'description': 'md5:60e5d30a78ced725f7b8d34370762941',
+            'upload_date': '20151124',
+            'timestamp': 1448354940,
+            'duration': 74022,
+            'view_count': int,
+        }
+    }
+    # extracted from https://audimedia.tv/assets/embed/embedded-player.js (dataSourceAuthToken)
+    _AUTH_TOKEN = 'e25b42847dba18c6c8816d5d8ce94c326e06823ebf0859ed164b3ba169be97f2'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        
+        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload');
+        _, stage_mode, video_id, lang = raw_payload.split('-')
+
+        # TODO: handle s and e stage_mode (live streams and ended live streams)
+        if stage_mode not in ('s', 'e'):
+            request = sanitized_Request(
+                'https://audimedia.tv/api/video/v1/videos/%s?embed[]=video_versions&embed[]=thumbnail_image&where[content_language_iso]=%s' % (video_id, lang),
+                headers={'X-Auth-Token': self._AUTH_TOKEN})
+            json_data = self._download_json(request, video_id)['results']
+            formats = []
+
+            stream_url_hls = json_data.get('stream_url_hls')
+            if stream_url_hls:
+                m3u8_formats = self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+
+            stream_url_hds = json_data.get('stream_url_hds')
+            if stream_url_hds:
+                f4m_formats = self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+
+            for video_version in json_data.get('video_versions'):
+                video_version_url = video_version.get('download_url') or video_version.get('stream_url')
+                if not video_version_url:
+                    continue
+                formats.append({
+                    'url': video_version_url,
+                    'width': int_or_none(video_version.get('width')),
+                    'height': int_or_none(video_version.get('height')),
+                    'abr': int_or_none(video_version.get('audio_bitrate')),
+                    'vbr': int_or_none(video_version.get('video_bitrate')),
+                })
+            self._sort_formats(formats)
+
+            return {
+                'id': video_id,
+                'title': json_data['title'],
+                'description': json_data.get('subtitle'),
+                'thumbnail': json_data.get('thumbnail_image', {}).get('file'),
+                'timestamp': parse_iso8601(json_data.get('publication_date')),
+                'duration': int_or_none(json_data.get('duration')),
+                'view_count': int_or_none(json_data.get('view_count')),
+                'formats': formats,
+            }

From 60121eb5141ab8ad527ec94373a180ecd5030d78 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 26 Nov 2015 22:43:31 +0100
Subject: [PATCH 1971/2145] [gameinformer] Add new extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/gameinformer.py | 42 ++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/gameinformer.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..4fbe57208 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -200,6 +200,7 @@ from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freevideo import FreeVideoIE
 from .funnyordie import FunnyOrDieIE
+from .gameinformer import GameInformerIE
 from .gamekings import GamekingsIE
 from .gameone import (
     GameOneIE,
diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
new file mode 100644
index 000000000..8bb764703
--- /dev/null
+++ b/youtube_dl/extractor/gameinformer.py
@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none
+
+
+class GameInformerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
+    _TEST = {
+        'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
+        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'info_dict': {
+            'id': '4515472681001',
+            'ext': 'm3u8',
+            'title': 'Replay - Animal Crossing',
+            'description': 'md5:2e211891b215c85d061adc7a4dd2d930',
+            'timestamp': 1443457610706,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        bc_api_url = self._search_regex(r"getVideo\('([^']+)'", webpage, 'brightcove api url')
+        json_data = self._download_json(bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions', display_id)
+
+        return {
+            'id': compat_str(json_data['id']),
+            'display_id': display_id,
+            'url': json_data['IOSRenditions'][0]['url'],
+            'title': json_data['name'],
+            'description': json_data.get('shortDescription'),
+            'timestamp': int_or_none(json_data.get('publishedDate')),
+            'duration': int_or_none(json_data.get('length')),
+        }

From 9bff48a0e7128b4d274a5dab4aaac8da0c900ede Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 Nov 2015 21:24:39 +0800
Subject: [PATCH 1972/2145] [options] Clarify --list-formats needs videos
 (closes #7669)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 079fe7e8a..359e8d300 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -338,7 +338,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
-        help='List all available formats')
+        help='List all available formats of specified videos')
     video_format.add_option(
         '--youtube-include-dash-manifest',
         action='store_true', dest='youtube_include_dash_manifest', default=True,

From 9d0e3668804fe91e77275e6bd859cbd0d784ab33 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 Nov 2015 21:37:45 +0800
Subject: [PATCH 1973/2145] [downloader/hls] Remove Accept-encoding from
 headers passed to ffmpeg

Fails for Youtube Gaming live streams (#7671)
---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 9a83a73dd..92765a3f9 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -35,7 +35,7 @@ class HlsFD(FileDownloader):
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
             args += [
                 '-headers',
-                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items())]
+                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items() if key.lower() != 'accept-encoding')]
 
         args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 

From bb6ac8369858e79b04e30e6ea25016500083628f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 27 Nov 2015 16:32:51 +0100
Subject: [PATCH 1974/2145] release 2015.11.27

---
 README.md             | 3 ++-
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b85e08e78..df419abe8 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific
                                      one is requested
-    -F, --list-formats               List all available formats
+    -F, --list-formats               List all available formats of specified
+                                     videos
     --youtube-skip-dash-manifest     Do not download the DASH manifests and
                                      related data on YouTube videos
     --merge-output-format FORMAT     If a merge is required (e.g.
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5ecff39fb..288b76e28 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.24'
+__version__ = '2015.11.27'

From 78a55d7a28662a65cba36c48eb69ecc4b150b336 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 27 Nov 2015 16:39:59 +0100
Subject: [PATCH 1975/2145] release 2015.11.27.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 288b76e28..bd0de9f53 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.27'
+__version__ = '2015.11.27.1'

From 313dfc45f500db3acc348d58f431197ce7c153a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 28 Nov 2015 01:07:07 +0100
Subject: [PATCH 1976/2145] [youtube] Ignore yt:stretch with zero width/height

---
 youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cfe9eed55..726b5ba0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -714,6 +714,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
             'only_matching': True,
         },
+        {
+            # Video with yt:stretch=17:0
+            'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
+            'info_dict': {
+                'id': 'Q39EVAstoRM',
+                'ext': 'mp4',
+                'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
+                'description': 'md5:ee18a25c350637c8faff806845bddee9',
+                'upload_date': '20151107',
+                'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
+                'uploader': 'CH GAMER DROID',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1496,10 +1512,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
             video_webpage)
         if stretched_m:
-            ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
-            for f in formats:
-                if f.get('vcodec') != 'none':
-                    f['stretched_ratio'] = ratio
+            w = float(stretched_m.group('w'))
+            h = float(stretched_m.group('h'))
+            if w > 0 and h > 0:
+                ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+                for f in formats:
+                    if f.get('vcodec') != 'none':
+                        f['stretched_ratio'] = ratio
 
         self._sort_formats(formats)
 

From 7ac40086f5f9fdc01c92d1b76acb742c661cf0ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 08:44:13 +0600
Subject: [PATCH 1977/2145] [dbtv] Expand _VALID_URL (Closes #7645)

---
 youtube_dl/extractor/dbtv.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 212217625..133cdc50b 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -13,8 +13,8 @@ from ..utils import (
 
 
 class DBTVIE(InfoExtractor):
-    _VALID_URL = r'http://dbtv\.no/(?P<id>[0-9]+)#(?P<display_id>.+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dbtv\.no/(?:(?:lazyplayer|player)/)?(?P<id>[0-9]+)(?:#(?P<display_id>.+))?'
+    _TESTS = [{
         'url': 'http://dbtv.no/3649835190001#Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
         'md5': 'b89953ed25dacb6edb3ef6c6f430f8bc',
         'info_dict': {
@@ -30,12 +30,18 @@ class DBTVIE(InfoExtractor):
             'view_count': int,
             'categories': list,
         }
-    }
+    }, {
+        'url': 'http://dbtv.no/3649835190001',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.dbtv.no/lazyplayer/4631135248001',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or video_id
 
         data = self._download_json(
             'http://api.dbtv.no/discovery/%s' % video_id, display_id)

From 4b3fbafdd261405c12e044a59d20554838c9c4c2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 28 Nov 2015 14:14:20 +0800
Subject: [PATCH 1978/2145] [options] Changed wording for --list-formats

As proposed by @dstftw at 9bff48a0e7128b4d274a5dab4aaac8da0c900ede
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 359e8d300..c46e136bf 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -338,7 +338,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
-        help='List all available formats of specified videos')
+        help='List all available formats of requested videos')
     video_format.add_option(
         '--youtube-include-dash-manifest',
         action='store_true', dest='youtube_include_dash_manifest', default=True,

From 41f24c321d04108a32f457d5d5445f2cfce705a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 28 Nov 2015 08:16:46 +0100
Subject: [PATCH 1979/2145] [youtube] Use the existing `w` and `h` variables

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 726b5ba0a..9da8d4bc5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1515,7 +1515,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             w = float(stretched_m.group('w'))
             h = float(stretched_m.group('h'))
             if w > 0 and h > 0:
-                ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+                ratio = w / h
                 for f in formats:
                     if f.get('vcodec') != 'none':
                         f['stretched_ratio'] = ratio

From 0cc71785461d273294f2ab96d50854172ba29c04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 11:48:18 +0100
Subject: [PATCH 1980/2145] [skynewsarabia] Add new extractor

---
 youtube_dl/extractor/__init__.py      |   4 +
 youtube_dl/extractor/skynewsarabia.py | 115 ++++++++++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 youtube_dl/extractor/skynewsarabia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..d981acb8e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -554,6 +554,10 @@ from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sina import SinaIE
+from .skynewsarabia import (
+    SkyNewsArabiaIE,
+    SkyNewsArabiaArticleIE,
+)
 from .slideshare import SlideshareIE
 from .slutload import SlutloadIE
 from .smotri import (
diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
new file mode 100644
index 000000000..3499b8cd5
--- /dev/null
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    parse_iso8601,
+    parse_duration,
+)
+
+
+class SkyNewArabiaBaseIE(InfoExtractor):
+    _IMAGE_BASE_URL = 'http://www.skynewsarabia.com/web/images'
+
+    def _call_api(self, path, value):
+        return self._download_json('http://api.skynewsarabia.com/web/rest/v2/%s/%s.json' % (path, value), value)
+
+    def _get_limelight_media_id(self, url):
+        return self._search_regex(r'/media/[^/]+/([a-z0-9]{32})', url, 'limelight media id')
+
+    def _get_image_url(self, image_path_template, width='1600', height='1200'):
+        return self._IMAGE_BASE_URL + image_path_template.format(width=width, height=height)
+
+    def _extract_video_info(self, video_data):
+        video_id = compat_str(video_data['id'])
+        return {
+            '_type': 'url_transparent',
+            'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
+            'id': video_id,
+            'title': video_data['headline'],
+            'description': video_data.get('summary'),
+            'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
+            'timestamp': parse_iso8601(video_data.get('date')),
+            'duration': parse_duration(video_data.get('runTime')),
+            'tags': video_data.get('tags'),
+            'categories': [video_data.get('topicTitle')],
+            'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
+            'ie_key': 'LimelightMedia',
+        }
+
+
+class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
+    _IE_NAME = 'skynewsarabia:video'
+    _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
+        'info_dict': {
+            'id': '794902',
+            'ext': 'flv',
+            'title': 'نصف مليون مصباح على شجرة كريسماس',
+            'description': 'md5:22f1b27f0850eeb10c7e59b1f16eb7c6',
+            'upload_date': '20151128',
+            'timestamp': 1448697198,
+            'duration': 2119,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._call_api('video', video_id)
+        return self._extract_video_info(video_data)
+
+
+class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
+    _IE_NAME = 'skynewsarabia:video'
+    _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
+        'info_dict': {
+            'id': '794549',
+            'ext': 'flv',
+            'title': 'بالفيديو.. ألعاب ذكية تحاكي واقع المنطقة',
+            'description': 'md5:0c373d29919a851e080ee4edd0c5d97f',
+            'upload_date': '20151126',
+            'timestamp': 1448559336,
+            'duration': 281.6,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.skynewsarabia.com/web/article/794844/%D8%A7%D8%B3%D8%AA%D9%87%D8%AF%D8%A7%D9%81-%D9%82%D9%88%D8%A7%D8%B1%D8%A8-%D8%A7%D9%94%D8%B3%D9%84%D8%AD%D8%A9-%D9%84%D9%85%D9%8A%D9%84%D9%8A%D8%B4%D9%8A%D8%A7%D8%AA-%D8%A7%D9%84%D8%AD%D9%88%D8%AB%D9%8A-%D9%88%D8%B5%D8%A7%D9%84%D8%AD',
+        'info_dict': {
+            'id': '794844',
+            'title': 'إحباط تهريب أسلحة لميليشيات الحوثي وصالح بجنوب اليمن',
+            'description': 'md5:5c927b8b2e805796e7f693538d96fc7e',
+        },
+        'playlist_mincount': 2,
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        article_data = self._call_api('article', article_id)
+        media_asset = article_data['mediaAsset']
+        if media_asset['type'] == 'VIDEO':
+            return {
+                '_type': 'url_transparent',
+                'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
+                'id': article_id,
+                'title': article_data['headline'],
+                'description': article_data.get('summary'),
+                'thumbnail': self._get_image_url(media_asset['imageUrl']),
+                'timestamp': parse_iso8601(article_data.get('date')),
+                'tags': article_data.get('tags'),
+                'categories': [article_data.get('topicTitle')],
+                'webpage_url': url,
+                'ie_key': 'LimelightMedia',
+            }
+        entries = [self._extract_video_info(item) for item in article_data.get('inlineItems', []) if item['type'] == 'VIDEO']
+        return self.playlist_result(entries, article_id, article_data['headline'], article_data.get('summary'))

From 4975650e002f3b9c20cc54940684c39c8f68e786 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 12:20:39 +0100
Subject: [PATCH 1981/2145] [skynewsarabia] fix IE_NAME

---
 youtube_dl/extractor/skynewsarabia.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index 3499b8cd5..d1c737020 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -40,7 +40,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
 
 
 class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
-    _IE_NAME = 'skynewsarabia:video'
+    IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
@@ -66,7 +66,7 @@ class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
 
 
 class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
-    _IE_NAME = 'skynewsarabia:video'
+    IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',

From 5faf9fed7e1c7922578467cfd48db5867ef9b91b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 18:50:21 +0600
Subject: [PATCH 1982/2145] [youtube] Clarify rationale for yt:stretch
 validation

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9da8d4bc5..1c2420a33 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1514,6 +1514,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if stretched_m:
             w = float(stretched_m.group('w'))
             h = float(stretched_m.group('h'))
+            # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
+            # We will only process correct ratios.
             if w > 0 and h > 0:
                 ratio = w / h
                 for f in formats:

From 9945c4994c8cdbfa9971df942332f2c75bc2734d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 20:21:03 +0600
Subject: [PATCH 1983/2145] Credit @reiv for soundcloud:search

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f465d20ed..cdb56de3b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -146,3 +146,4 @@ Lukáš Lalinský
 Qijiang Fan
 Rémy Léone
 Marco Ferragina
+reiv

From 7ad4258add964b2604ada611d380b5594126d85c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:39:36 +0600
Subject: [PATCH 1984/2145] [bloomberg] Relax _VALID_URL even more (Closes
 #7685)

---
 youtube_dl/extractor/bloomberg.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 11ace91dd..8b2f94211 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/news/[^/]+/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?bloomberg\.com/(?:[^/]+/)*(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
@@ -20,6 +20,9 @@ class BloombergIE(InfoExtractor):
     }, {
         'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
         'only_matching': True,
+    }, {
+        'url': 'http://www.bloomberg.com/politics/videos/2015-11-25/karl-rove-on-jeb-bush-s-struggles-stopping-trump',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9a4f12be98a0d59de9b5da65d2cc24b04fe26929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:40:29 +0600
Subject: [PATCH 1985/2145] [bloomberg] Modernize

---
 youtube_dl/extractor/bloomberg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 8b2f94211..b6197d3f1 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -35,7 +35,7 @@ class BloombergIE(InfoExtractor):
             'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
         formats = []
         for stream in embed_info['streams']:
-            if stream["muxing_format"] == "TS":
+            if stream['muxing_format'] == 'TS':
                 formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
             else:
                 formats.extend(self._extract_f4m_formats(stream['url'], video_id))

From 4191fdf147729728ca9c487f25ca66bf3d037cd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:41:39 +0600
Subject: [PATCH 1986/2145] [bloomberg] Improve video id regex

---
 youtube_dl/extractor/bloomberg.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index b6197d3f1..c28ed0c79 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -28,7 +28,9 @@ class BloombergIE(InfoExtractor):
     def _real_extract(self, url):
         name = self._match_id(url)
         webpage = self._download_webpage(url, name)
-        video_id = self._search_regex(r'"bmmrId":"(.+?)"', webpage, 'id')
+        video_id = self._search_regex(
+            r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'id', group='url')
         title = re.sub(': Video$', '', self._og_search_title(webpage))
 
         embed_info = self._download_json(

From b7faebbac87430106d0249a09fcca55a5ab112de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:45:19 +0600
Subject: [PATCH 1987/2145] [bloomberg] Improve formats extraction

---
 youtube_dl/extractor/bloomberg.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index c28ed0c79..ebeef8f2a 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -37,10 +37,19 @@ class BloombergIE(InfoExtractor):
             'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
         formats = []
         for stream in embed_info['streams']:
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
             if stream['muxing_format'] == 'TS':
-                formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
+                m3u8_formats = self._extract_m3u8_formats(
+                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
-                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+                f4m_formats = self._extract_f4m_formats(
+                    stream_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
         self._sort_formats(formats)
 
         return {

From 50e12e9df186b475d0e500f61426d9e72d5b4320 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 18:10:37 +0100
Subject: [PATCH 1988/2145] [acast] Add new extractor

---
 youtube_dl/extractor/__init__.py |  4 ++
 youtube_dl/extractor/acast.py    | 66 ++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/acast.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..f7594832f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 from .abc import ABCIE
 from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
+from .acast import (
+    ACastIE,
+    ACastChannelIE,
+)
 from .addanime import AddAnimeIE
 from .adobetv import (
     AdobeTVIE,
diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
new file mode 100644
index 000000000..c2719ca32
--- /dev/null
+++ b/youtube_dl/extractor/acast.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none
+
+
+class ACastBaseIE(InfoExtractor):
+    _API_BASE_URL = 'https://www.acast.com/api/'
+
+
+class ACastIE(ACastBaseIE):
+    IE_NAME = 'acast'
+    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
+    _TEST = {
+        'url': 'https://www.acast.com/gardenersquestiontime/liverpool',
+        'md5': '9e9cd59c3a8a7d8d5407605f51093050',
+        'info_dict': {
+            'id': '43da2262-ade7-420c-8564-f6367da7c010',
+            'ext': 'mp3',
+            'title': 'Liverpool',
+            'timestamp': 1446163200000,
+            'description': 'md5:170432c9956eec0670d7080a75000d5b',
+            'duration': 2520,
+        }
+    }
+
+    def _real_extract(self, url):
+        channel, display_id = re.match(self._VALID_URL, url).groups()
+        cast_data = self._download_json(self._API_BASE_URL + 'channels/%s/acasts/%s/playback' % (channel, display_id), display_id)
+
+        return {
+            'id': compat_str(cast_data['id']),
+            'display_id': display_id,
+            'url': cast_data['blings'][0]['audio'],
+            'title': cast_data['name'],
+            'description': cast_data.get('description'),
+            'thumbnail': cast_data.get('image'),
+            'timestamp': int_or_none(cast_data.get('publishingDate')),
+            'duration': int_or_none(cast_data.get('duration')),
+        }
+
+
+class ACastChannelIE(ACastBaseIE):
+    IE_NAME = 'acast:channel'
+    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
+    _TEST = {
+        'url': 'https://www.acast.com/gardenersquestiontime',
+        'info_dict': {
+            'id': '9d8f6f73-6b9d-4d16-9399-52bf88f8f611',
+            'title': 'Gardeners\' Question Time',
+            'description': 'md5:c7ef18049da6a52b63d371b3edccce90',
+        },
+        'playlist_mincount': 5,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        channel_data = self._download_json(self._API_BASE_URL + 'channels/%s' % display_id, display_id)
+        casts = self._download_json(self._API_BASE_URL + 'channels/%s/acasts' % display_id, display_id)
+        entries = [self.url_result('https://www.acast.com/%s/%s' % (display_id, cast['url']), 'ACast') for cast in casts]
+
+        return self.playlist_result(entries, compat_str(channel_data['id']), channel_data['name'], channel_data.get('description'))

From f4c7ef98624bd407a8f8f91215cb95a2c5db9472 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 18:20:44 +0100
Subject: [PATCH 1989/2145] [skynewsarabia] return empty categories array if
 there is no topic

---
 youtube_dl/extractor/skynewsarabia.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index d1c737020..f09fee102 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -23,6 +23,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
 
     def _extract_video_info(self, video_data):
         video_id = compat_str(video_data['id'])
+        topic = video_data.get('topicTitle')
         return {
             '_type': 'url_transparent',
             'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
@@ -32,8 +33,8 @@ class SkyNewArabiaBaseIE(InfoExtractor):
             'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
             'timestamp': parse_iso8601(video_data.get('date')),
             'duration': parse_duration(video_data.get('runTime')),
-            'tags': video_data.get('tags'),
-            'categories': [video_data.get('topicTitle')],
+            'tags': video_data.get('tags', []),
+            'categories': [topic] if topic else [],
             'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
             'ie_key': 'LimelightMedia',
         }
@@ -98,6 +99,7 @@ class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
         article_data = self._call_api('article', article_id)
         media_asset = article_data['mediaAsset']
         if media_asset['type'] == 'VIDEO':
+            topic = article_data.get('topicTitle')
             return {
                 '_type': 'url_transparent',
                 'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
@@ -106,8 +108,8 @@ class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
                 'description': article_data.get('summary'),
                 'thumbnail': self._get_image_url(media_asset['imageUrl']),
                 'timestamp': parse_iso8601(article_data.get('date')),
-                'tags': article_data.get('tags'),
-                'categories': [article_data.get('topicTitle')],
+                'tags': article_data.get('tags', []),
+                'categories': [topic] if topic else [],
                 'webpage_url': url,
                 'ie_key': 'LimelightMedia',
             }

From 2a776f978849e0c66f70133747e7fd244f516f7f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 20:22:31 +0100
Subject: [PATCH 1990/2145] [cspan] change into a function

---
 youtube_dl/extractor/cspan.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 388460a32..7b685d157 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -56,9 +56,6 @@ class CSpanIE(InfoExtractor):
         }
     }]
 
-    def get_text_attr(self, d, attr):
-        return d.get(attr, {}).get('#text')
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -74,11 +71,14 @@ class CSpanIE(InfoExtractor):
                 surl = smuggle_url(senate_isvp_url, {'force_title': title})
                 return self.url_result(surl, 'SenateISVP', video_id, title)
 
+        def get_text_attr(d, attr):
+            return d.get(attr, {}).get('#text')
+
         data = self._download_json(
             'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
             video_id)['video']
         if data['@status'] != 'Success':
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, self.get_text_attr(data, 'error')), expected=True)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
 
         doc = self._download_xml(
             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -90,17 +90,17 @@ class CSpanIE(InfoExtractor):
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
         files = data['files']
-        capfile = self.get_text_attr(data, 'capfile')
+        capfile = get_text_attr(data, 'capfile')
 
         entries = []
         for partnum, f in enumerate(files):
             formats = []
             for quality in f['qualities']:
                 formats.append({
-                    'format_id': '%s-%sp' % (self.get_text_attr(quality, 'bitrate'), self.get_text_attr(quality, 'height')),
-                    'url': unescapeHTML(self.get_text_attr(quality, 'file')),
-                    'height': int_or_none(self.get_text_attr(quality, 'height')),
-                    'tbr': int_or_none(self.get_text_attr(quality, 'bitrate')),
+                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(get_text_attr(quality, 'file')),
+                    'height': int_or_none(get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                 })
             self._sort_formats(formats)
             entries.append({
@@ -111,7 +111,7 @@ class CSpanIE(InfoExtractor):
                 'formats': formats,
                 'description': description,
                 'thumbnail': thumbnail,
-                'duration': int_or_none(self.get_text_attr(f, 'length')),
+                'duration': int_or_none(get_text_attr(f, 'length')),
                 'subtitles': {
                     'en': [{
                         'url': capfile,

From 87f0e62d94e0486598d123e26db3173e6f1d18e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:42:50 +0800
Subject: [PATCH 1991/2145] [utils] Separate codes for handling Youtubedl-*
 headers

---
 youtube_dl/utils.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d7b737e21..653a49055 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -663,6 +663,15 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     return hc
 
 
+def handle_youtubedl_headers(headers):
+    if 'Youtubedl-no-compression' in headers:
+        filtered_headers = dict((k, v) for k, v in headers.items() if k.lower() != 'accept-encoding')
+        del filtered_headers['Youtubedl-no-compression']
+        return filtered_headers
+
+    return headers
+
+
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -731,10 +740,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             # The dict keys are capitalized because of this bug by urllib
             if h.capitalize() not in req.headers:
                 req.add_header(h, v)
-        if 'Youtubedl-no-compression' in req.headers:
-            if 'Accept-encoding' in req.headers:
-                del req.headers['Accept-encoding']
-            del req.headers['Youtubedl-no-compression']
+
+        req.headers = handle_youtubedl_headers(req.headers)
 
         if sys.version_info < (2, 7) and '#' in req.get_full_url():
             # Python 2.6 is brain-dead when it comes to fragments

From 94e8c8047353eb541fa20dcf55819cc6ee6d3303 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:43:59 +0800
Subject: [PATCH 1992/2145] [downloader/hls] Respect Youtubedl-* headers

---
 youtube_dl/downloader/hls.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 92765a3f9..b5a3e1167 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -13,6 +13,7 @@ from ..utils import (
     encodeArgument,
     encodeFilename,
     sanitize_open,
+    handle_youtubedl_headers,
 )
 
 
@@ -33,9 +34,10 @@ class HlsFD(FileDownloader):
         if info_dict['http_headers'] and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            headers = handle_youtubedl_headers(info_dict['http_headers'])
             args += [
                 '-headers',
-                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items() if key.lower() != 'accept-encoding')]
+                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
         args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 

From ac5a69af45307b583a9a6088abe5939bec18d562 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:44:24 +0800
Subject: [PATCH 1993/2145] [youtube] Disable compression for live streams

---
 youtube_dl/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1c2420a33..52f4fe36d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1475,6 +1475,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             manifest_url = video_info['hlsvp'][0]
             url_map = self._extract_from_m3u8(manifest_url, video_id)
             formats = _map_to_format_list(url_map)
+            # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
+            for a_format in formats:
+                if 'http_headers' not in a_format:
+                    a_format['http_headers'] = {}
+                a_format['http_headers']['Youtubedl-no-compression'] = True
         else:
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 

From 0424ec307bb920a2a7c217a741241f3d2af84efa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:46:04 +0800
Subject: [PATCH 1994/2145] [utils] Correct docstring of YoutubeDLHandler

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 653a49055..c43e9e3a1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -679,7 +679,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     the standard headers to every HTTP request and handles gzipped and
     deflated responses from web servers. If compression is to be avoided in
     a particular request, the original request in the program code only has
-    to include the HTTP header "Youtubedl-No-Compression", which will be
+    to include the HTTP header "Youtubedl-no-compression", which will be
     removed before making the real request.
 
     Part of this code was copied from:

From 8639f89f516c5bd1e4fda38c40e2a5a9b940ad85 Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 28 Nov 2015 22:56:24 -0600
Subject: [PATCH 1995/2145] Always use PYTHON env var in Makefile

---
 Makefile | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..337a2eefb 100644
--- a/Makefile
+++ b/Makefile
@@ -60,34 +60,34 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	chmod a+x youtube-dl
 
 README.md: youtube_dl/*.py youtube_dl/*/*.py
-	COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py
+	COLUMNS=80 $(PYTHON) youtube_dl/__main__.py --help | $(PYTHON) devscripts/make_readme.py
 
 CONTRIBUTING.md: README.md
-	python devscripts/make_contributing.py README.md CONTRIBUTING.md
+	$(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
 
 supportedsites:
-	python devscripts/make_supportedsites.py docs/supportedsites.md
+	$(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md
 
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
 youtube-dl.1: README.md
-	python devscripts/prepare_manpage.py >youtube-dl.1.temp.md
+	$(PYTHON) devscripts/prepare_manpage.py >youtube-dl.1.temp.md
 	pandoc -s -f markdown -t man youtube-dl.1.temp.md -o youtube-dl.1
 	rm -f youtube-dl.1.temp.md
 
 youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in
-	python devscripts/bash-completion.py
+	$(PYTHON) devscripts/bash-completion.py
 
 bash-completion: youtube-dl.bash-completion
 
 youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in
-	python devscripts/zsh-completion.py
+	$(PYTHON) devscripts/zsh-completion.py
 
 zsh-completion: youtube-dl.zsh
 
 youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
-	python devscripts/fish-completion.py
+	$(PYTHON) devscripts/fish-completion.py
 
 fish-completion: youtube-dl.fish
 

From 992fc9d6e124b910ff3d720e252ef9aad99b2a8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:58:29 +0800
Subject: [PATCH 1996/2145] [utils] Refactor handle_youtubedl_headers for
 future extension

---
 youtube_dl/utils.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c43e9e3a1..d0606b4bc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -664,12 +664,13 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
 
 
 def handle_youtubedl_headers(headers):
-    if 'Youtubedl-no-compression' in headers:
-        filtered_headers = dict((k, v) for k, v in headers.items() if k.lower() != 'accept-encoding')
-        del filtered_headers['Youtubedl-no-compression']
-        return filtered_headers
+    filtered_headers = headers
 
-    return headers
+    if 'Youtubedl-no-compression' in filtered_headers:
+        filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
+        del filtered_headers['Youtubedl-no-compression']
+
+    return filtered_headers
 
 
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):

From bf2c8c8f82ff54f9594673c48e531661a72dbdcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 17:03:33 +0600
Subject: [PATCH 1997/2145] [spiegel] Fix extraction (Closes #7693)

---
 youtube_dl/extractor/spiegel.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 5bd3c0087..39a7aaf9d 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -58,7 +58,8 @@ class SpiegelIE(InfoExtractor):
         description = self._html_search_meta('description', webpage, 'description')
 
         base_url = self._search_regex(
-            r'var\s+server\s*=\s*"([^"]+)\"', webpage, 'server URL')
+            [r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
+            webpage, 'server URL', group='url')
 
         xml_url = base_url + video_id + '.xml'
         idoc = self._download_xml(xml_url, video_id)

From 049d71d8745014bf5ec23e25e51d6b92556baa8c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 19:52:48 +0800
Subject: [PATCH 1998/2145] [youtube] Simplify and make sure header values are
 strings

---
 youtube_dl/extractor/youtube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 52f4fe36d..4f375e2c8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1477,9 +1477,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             formats = _map_to_format_list(url_map)
             # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
             for a_format in formats:
-                if 'http_headers' not in a_format:
-                    a_format['http_headers'] = {}
-                a_format['http_headers']['Youtubedl-no-compression'] = True
+                a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
         else:
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 

From 040ac686798fdc922157cca64d654933e3f6d096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 21:01:59 +0600
Subject: [PATCH 1999/2145] [youtube] Extend _VALID_URL (Closes #7694)

---
 youtube_dl/extractor/youtube.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4f375e2c8..55a06eb68 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -258,7 +258,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
-                                 (?:.*?&)??                                   # any other preceding param (like /?s=tuff&v=xxxx)
+                                 (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
                                  v=
                              )
                          ))
@@ -730,6 +730,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
+            'only_matching': True,
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From 2e1b92854000662e554413df0c34c1cbc0d7ffc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 21:04:11 +0600
Subject: [PATCH 2000/2145] [youtube:playlist] Extend _VALID_URL

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 55a06eb68..032691e7f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1566,7 +1566,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtract
                         youtube\.com/
                         (?:
                            (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
-                           \? (?:.*?&)*? (?:p|a|list)=
+                           \? (?:.*?[&;])*? (?:p|a|list)=
                         |  p/
                         )
                         (

From d53a4af1a49413a38d639aeb7f522c4ebff8f5c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 03:47:01 +0600
Subject: [PATCH 2001/2145] [pornhub:playlist] Allow alphanumeric viewkeys
 (Closes #7695)

---
 youtube_dl/extractor/pornhub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 965940a4b..08275687d 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -147,7 +147,8 @@ class PornHubPlaylistIE(InfoExtractor):
 
         entries = [
             self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
-            for video_url in set(re.findall('href="/?(view_video\.php\?viewkey=\d+[^"]*)"', webpage))
+            for video_url in set(re.findall(
+                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
         ]
 
         playlist = self._parse_json(

From af284305d58a9915a8ef00d056484b3a59548dda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 03:58:39 +0600
Subject: [PATCH 2002/2145] [vodlocker] Capture file not found error (Closes
 #7696)

---
 youtube_dl/extractor/vodlocker.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index be0a2780f..357594a11 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -3,11 +3,14 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import sanitized_Request
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
+)
 
 
 class VodlockerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
+    _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
 
     _TESTS = [{
         'url': 'http://vodlocker.com/e8wvyzz4sl42',
@@ -24,6 +27,12 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if any(p in webpage for p in (
+                '>THIS FILE WAS DELETED<',
+                '>File Not Found<',
+                'The file you were looking for could not be found, sorry for any inconvenience.<')):
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':

From 59ee8a86471af488c2ee16dcacf7a913636f0150 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 20:10:09 +0600
Subject: [PATCH 2003/2145] [facebook] Make alternative title optional (Closes
 #7700)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index fd854411b..321eec59e 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -164,7 +164,7 @@ class FacebookIE(InfoExtractor):
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
-                webpage, 'alternative title', fatal=False)
+                webpage, 'alternative title', default=None)
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id

From 4c6b4764f0260808d321cfb6cca1daa5e3eb13d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 20:42:05 +0600
Subject: [PATCH 2004/2145] [youtube] Clarify itag 272 possible resolutions
 (#7699)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 032691e7f..9b39505ba 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -346,6 +346,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},

From 874ae0354e96a578dc44846fffd91b01079ccf1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 18:35:24 +0600
Subject: [PATCH 2005/2145] [nrk] Extract f4m formats and impose geo
 restriction only when not media URL (Closes #7715)

---
 youtube_dl/extractor/nrk.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 8ac38a174..6ff13050d 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     parse_duration,
@@ -48,12 +49,22 @@ class NRKIE(InfoExtractor):
             'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
             video_id, 'Downloading media JSON')
 
-        if data['usageRights']['isGeoBlocked']:
-            raise ExtractorError(
-                'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
-                expected=True)
+        media_url = data.get('mediaUrl')
 
-        video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
+        if not media_url:
+            if data['usageRights']['isGeoBlocked']:
+                raise ExtractorError(
+                    'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
+                    expected=True)
+
+        if determine_ext(media_url) == 'f4m':
+            formats = self._extract_f4m_formats(
+                media_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id, f4m_id='hds')
+        else:
+            formats = [{
+                'url': media_url,
+                'ext': 'flv',
+            }]
 
         duration = parse_duration(data.get('duration'))
 
@@ -67,12 +78,11 @@ class NRKIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'flv',
             'title': data['title'],
             'description': data['description'],
             'duration': duration,
             'thumbnail': thumbnail,
+            'formats': formats,
         }
 
 

From 3b35c3425e2c4d9836a3efe5cf73e35a60f674d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 20:35:46 +0600
Subject: [PATCH 2006/2145] [udemy] Extract formats from data.outputs (#7704)

---
 youtube_dl/extractor/udemy.py | 46 ++++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 825172806..9d31a3f5b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     sanitized_Request,
 )
 
@@ -127,26 +128,47 @@ class UdemyIE(InfoExtractor):
 
         video_id = asset['id']
         thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
-        duration = asset['data']['duration']
+        duration = int_or_none(asset.get('data', {}).get('duration'))
 
         download_url = asset.get('downloadUrl') or asset.get('download_url')
 
         video = download_url.get('Video') or download_url.get('video')
         video_480p = download_url.get('Video480p') or download_url.get('video_480p')
 
-        formats = [
-            {
-                'url': video_480p[0],
-                'format_id': '360p',
-            },
-            {
-                'url': video[0],
-                'format_id': '720p',
-            },
-        ]
+        formats = [{
+            'url': video_480p[0],
+            'format_id': 'download-360p',
+        }, {
+            'url': video[0],
+            'format_id': 'download-720p',
+        }]
+
+        # Some videos also contain formats in asset['data']['outputs'] (e.g.
+        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+        outputs = asset.get('data', {}).get('outputs')
+        if isinstance(outputs, dict):
+            for format_id, f in outputs.items():
+                video_url = f.get('url')
+                if video_url:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': '%sp' % (f.get('labe1l') or format_id),
+                        'width': int_or_none(f.get('width')),
+                        'height': int_or_none(f.get('height')),
+                        'vbr': int_or_none(f.get('video_bitrate_in_kbps')),
+                        'vcodec': f.get('video_codec'),
+                        'fps': int_or_none(f.get('frame_rate')),
+                        'abr': int_or_none(f.get('audio_bitrate_in_kbps')),
+                        'acodec': f.get('audio_codec'),
+                        'asr': int_or_none(f.get('audio_sample_rate')),
+                        'tbr': int_or_none(f.get('total_bitrate_in_kbps')),
+                        'filesize': int_or_none(f.get('file_size_in_bytes')),
+                    })
+
+        self._sort_formats(formats)
 
         title = lecture['title']
-        description = lecture['description']
+        description = lecture.get('description')
 
         return {
             'id': video_id,

From 78717fc32802cb1e0f240cfbdf02207c6356ff2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 22:10:10 +0600
Subject: [PATCH 2007/2145] [udemy] Allow authentication via cookies

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 9d31a3f5b..e743c0262 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -72,7 +72,7 @@ class UdemyIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            self.raise_login_required('Udemy account is required')
+            return
 
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')

From 328f82d59a0d3b743280c3ab532949c9220383ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:48:27 +0600
Subject: [PATCH 2008/2145] [udemy] Semi-switch to api 2.0 (Closes #7704)

* Use api 2.0 to get lectures since it provides more formats
* Fix authorization for api 2.0
* Autotry enrolling in the course for single lectures
* Extract additional metadata rom asset['data']['outputs']
---
 youtube_dl/extractor/udemy.py | 160 ++++++++++++++++++++--------------
 1 file changed, 95 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e743c0262..26ae8cc36 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
     int_or_none,
     sanitized_Request,
 )
@@ -19,6 +20,8 @@ class UdemyIE(InfoExtractor):
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
     _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
     _ORIGIN_URL = 'https://www.udemy.com'
+    _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
+    _ALREADY_ENROLLED = '>You are already taking this course.<'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -34,6 +37,29 @@ class UdemyIE(InfoExtractor):
         'skip': 'Requires udemy account credentials',
     }]
 
+    def _enroll_course(self, webpage, course_id):
+        enroll_url = self._search_regex(
+            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
+            webpage, 'enroll url', group='url',
+            default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id)
+        webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
+        if self._SUCCESSFULLY_ENROLLED in webpage:
+            self.to_screen('%s: Successfully enrolled in' % course_id)
+        elif self._ALREADY_ENROLLED in webpage:
+            self.to_screen('%s: Already enrolled in' % course_id)
+
+    def _download_lecture(self, course_id, lecture_id):
+        return self._download_json(
+            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % (
+                course_id, lecture_id, compat_urllib_parse.urlencode({
+                    'video_only': '',
+                    'auto_play': '',
+                    'fields[lecture]': 'title,description,asset',
+                    'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
+                    'instructorPreviewMode': 'False',
+                })),
+            lecture_id, 'Downloading lecture JSON', fatal=False)
+
     def _handle_error(self, response):
         if not isinstance(response, dict):
             return
@@ -45,7 +71,7 @@ class UdemyIE(InfoExtractor):
                 error_str += ' - %s' % error_data.get('formErrors')
             raise ExtractorError(error_str, expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
         headers = {
             'X-Udemy-Snail-Case': 'true',
             'X-Requested-With': 'XMLHttpRequest',
@@ -55,6 +81,7 @@ class UdemyIE(InfoExtractor):
                 headers['X-Udemy-Client-Id'] = cookie.value
             elif cookie.name == 'access_token':
                 headers['X-Udemy-Bearer-Token'] = cookie.value
+                headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value
 
         if isinstance(url_or_request, compat_urllib_request.Request):
             for header, value in headers.items():
@@ -62,7 +89,7 @@ class UdemyIE(InfoExtractor):
         else:
             url_or_request = sanitized_Request(url_or_request, headers=headers)
 
-        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
+        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs)
         self._handle_error(response)
         return response
 
@@ -110,66 +137,77 @@ class UdemyIE(InfoExtractor):
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
 
-        lecture = self._download_json(
-            'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
-            lecture_id, 'Downloading lecture JSON')
+        webpage = self._download_webpage(url, lecture_id)
 
-        asset_type = lecture.get('assetType') or lecture.get('asset_type')
+        course_id = self._search_regex(
+            r'data-course-id=["\'](\d+)', webpage, 'course id')
+
+        try:
+            lecture = self._download_lecture(course_id, lecture_id)
+        except ExtractorError as e:
+            # Error could possibly mean we are not enrolled in the course
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                self._enroll_course(webpage, course_id)
+                lecture_id = self._download_lecture(course_id, lecture_id)
+            else:
+                raise
+
+        title = lecture['title']
+        description = lecture.get('description')
+
+        asset = lecture['asset']
+
+        asset_type = asset.get('assetType') or asset.get('asset_type')
         if asset_type != 'Video':
             raise ExtractorError(
                 'Lecture %s is not a video' % lecture_id, expected=True)
 
-        asset = lecture['asset']
-
         stream_url = asset.get('streamUrl') or asset.get('stream_url')
-        mobj = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url)
-        if mobj:
-            return self.url_result(mobj.group(1), 'Youtube')
+        if stream_url:
+            youtube_url = self._search_regex(
+                r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
+            if youtube_url:
+                return self.url_result(youtube_url, 'Youtube')
 
         video_id = asset['id']
         thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
-        duration = int_or_none(asset.get('data', {}).get('duration'))
+        duration = float_or_none(asset.get('data', {}).get('duration'))
+        outputs = asset.get('data', {}).get('outputs', {})
 
-        download_url = asset.get('downloadUrl') or asset.get('download_url')
-
-        video = download_url.get('Video') or download_url.get('video')
-        video_480p = download_url.get('Video480p') or download_url.get('video_480p')
-
-        formats = [{
-            'url': video_480p[0],
-            'format_id': 'download-360p',
-        }, {
-            'url': video[0],
-            'format_id': 'download-720p',
-        }]
-
-        # Some videos also contain formats in asset['data']['outputs'] (e.g.
-        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-        outputs = asset.get('data', {}).get('outputs')
-        if isinstance(outputs, dict):
-            for format_id, f in outputs.items():
-                video_url = f.get('url')
-                if video_url:
-                    formats.append({
-                        'url': video_url,
-                        'format_id': '%sp' % (f.get('labe1l') or format_id),
-                        'width': int_or_none(f.get('width')),
-                        'height': int_or_none(f.get('height')),
-                        'vbr': int_or_none(f.get('video_bitrate_in_kbps')),
-                        'vcodec': f.get('video_codec'),
-                        'fps': int_or_none(f.get('frame_rate')),
-                        'abr': int_or_none(f.get('audio_bitrate_in_kbps')),
-                        'acodec': f.get('audio_codec'),
-                        'asr': int_or_none(f.get('audio_sample_rate')),
-                        'tbr': int_or_none(f.get('total_bitrate_in_kbps')),
-                        'filesize': int_or_none(f.get('file_size_in_bytes')),
+        formats = []
+        for format_ in asset.get('download_urls', {}).get('Video', []):
+            video_url = format_.get('file')
+            if not video_url:
+                continue
+            format_id = format_.get('label')
+            f = {
+                'url': format_['file'],
+                'height': int_or_none(format_id),
+            }
+            if format_id:
+                # Some videos contain additional metadata (e.g.
+                # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+                output = outputs.get(format_id)
+                if isinstance(output, dict):
+                    f.update({
+                        'format_id': '%sp' % (output.get('label') or format_id),
+                        'width': int_or_none(output.get('width')),
+                        'height': int_or_none(output.get('height')),
+                        'vbr': int_or_none(output.get('video_bitrate_in_kbps')),
+                        'vcodec': output.get('video_codec'),
+                        'fps': int_or_none(output.get('frame_rate')),
+                        'abr': int_or_none(output.get('audio_bitrate_in_kbps')),
+                        'acodec': output.get('audio_codec'),
+                        'asr': int_or_none(output.get('audio_sample_rate')),
+                        'tbr': int_or_none(output.get('total_bitrate_in_kbps')),
+                        'filesize': int_or_none(output.get('file_size_in_bytes')),
                     })
+                else:
+                    f['format_id'] = '%sp' % format_id
+            formats.append(f)
 
         self._sort_formats(formats)
 
-        title = lecture['title']
-        description = lecture.get('description')
-
         return {
             'id': video_id,
             'title': title,
@@ -182,9 +220,7 @@ class UdemyIE(InfoExtractor):
 
 class UdemyCourseIE(UdemyIE):
     IE_NAME = 'udemy:course'
-    _VALID_URL = r'https?://www\.udemy\.com/(?P<coursepath>[\da-z-]+)'
-    _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
-    _ALREADY_ENROLLED = '>You are already taking this course.<'
+    _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[\da-z-]+)'
     _TESTS = []
 
     @classmethod
@@ -192,24 +228,18 @@ class UdemyCourseIE(UdemyIE):
         return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        course_path = mobj.group('coursepath')
+        course_path = self._match_id(url)
+
+        webpage = self._download_webpage(url, course_path)
 
         response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s' % course_path,
             course_path, 'Downloading course JSON')
 
-        course_id = int(response['id'])
-        course_title = response['title']
+        course_id = response['id']
+        course_title = response.get('title')
 
-        webpage = self._download_webpage(
-            'https://www.udemy.com/course/subscribe/?courseId=%s' % course_id,
-            course_id, 'Enrolling in the course')
-
-        if self._SUCCESSFULLY_ENROLLED in webpage:
-            self.to_screen('%s: Successfully enrolled in' % course_id)
-        elif self._ALREADY_ENROLLED in webpage:
-            self.to_screen('%s: Already enrolled in' % course_id)
+        self._enroll_course(webpage, course_id)
 
         response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,

From 9fc87fa767425ee7d56b2ce4650c9b06b480e005 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:51:47 +0600
Subject: [PATCH 2009/2145] [udemy] Remove unused import

---
 youtube_dl/extractor/udemy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 26ae8cc36..2a54f3764 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,7 +5,6 @@ from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
-    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,

From 24121bc703152312cfbb70f01ebd39e2fe1197e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:53:03 +0600
Subject: [PATCH 2010/2145] [udemy] Make lecture downloading fatal

---
 youtube_dl/extractor/udemy.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 2a54f3764..59832b1ec 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -57,7 +57,7 @@ class UdemyIE(InfoExtractor):
                     'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
                     'instructorPreviewMode': 'False',
                 })),
-            lecture_id, 'Downloading lecture JSON', fatal=False)
+            lecture_id, 'Downloading lecture JSON')
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -70,7 +70,7 @@ class UdemyIE(InfoExtractor):
                 error_str += ' - %s' % error_data.get('formErrors')
             raise ExtractorError(error_str, expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
         headers = {
             'X-Udemy-Snail-Case': 'true',
             'X-Requested-With': 'XMLHttpRequest',
@@ -88,7 +88,7 @@ class UdemyIE(InfoExtractor):
         else:
             url_or_request = sanitized_Request(url_or_request, headers=headers)
 
-        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs)
+        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
         self._handle_error(response)
         return response
 

From 22d7368dfb384e7698faad6d2891b4aaceab3d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 02:34:31 +0600
Subject: [PATCH 2011/2145] [bbc] Extract _ID_REGEX and ad one more video id
 pattern (Closes #7724)

---
 youtube_dl/extractor/bbc.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 33b296eaf..5f7265a06 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -22,7 +22,8 @@ from ..compat import (
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>[\da-z]{8})'
+    _ID_REGEX = r'[pb][\da-z]{7}'
+    _VALID_URL = r'https?://(?:(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])|)(?P<id>%s)' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -465,7 +466,7 @@ class BBCCoUkIE(InfoExtractor):
 
         if not programme_id:
             programme_id = self._search_regex(
-                r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+                r'"vpid"\s*:\s*"(%s)"' % self._ID_REGEX, webpage, 'vpid', fatal=False, default=None)
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
@@ -780,8 +781,9 @@ class BBCIE(BBCCoUkIE):
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(
-            [r'data-video-player-vpid="([\da-z]{8})"',
-             r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+            [r'data-video-player-vpid="(%s)"' % self._ID_REGEX,
+             r'<param[^>]+name="externalIdentifier"[^>]+value="(%s)"' % self._ID_REGEX,
+             r'videoId\s*:\s*["\'](%s)["\']' % self._ID_REGEX],
             webpage, 'vpid', default=None)
 
         if programme_id:
@@ -816,7 +818,7 @@ class BBCIE(BBCCoUkIE):
 
         # Multiple video article (e.g.
         # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
-        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+%s(?:\b[^"]+)?' % self._ID_REGEX
         entries = []
         for match in extract_all(r'new\s+SMP\(({.+?})\)'):
             embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')

From 7b1e379ca9316433ee911f2f36c7808522fff13d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 3 Dec 2015 13:47:21 +0100
Subject: [PATCH 2012/2145] [gametrailers] Fix extraction (fixes #7722)

They have stopped using the MTV system.
---
 youtube_dl/extractor/gametrailers.py | 61 ++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py
index a6ab795ae..c3f031d9c 100644
--- a/youtube_dl/extractor/gametrailers.py
+++ b/youtube_dl/extractor/gametrailers.py
@@ -1,19 +1,62 @@
 from __future__ import unicode_literals
 
-from .mtv import MTVServicesInfoExtractor
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    url_basename,
+)
 
 
-class GametrailersIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'http://www\.gametrailers\.com/(?P<type>videos|reviews|full-episodes)/(?P<id>.*?)/(?P<title>.*)'
+class GametrailersIE(InfoExtractor):
+    _VALID_URL = r'http://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
+
     _TEST = {
-        'url': 'http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer',
-        'md5': '4c8e67681a0ea7ec241e8c09b3ea8cf7',
+        'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
+        'md5': 'f28c4efa0bdfaf9b760f6507955b6a6a',
         'info_dict': {
-            'id': '70e9a5d7-cf25-4a10-9104-6f3e7342ae0d',
+            'id': '2983958',
             'ext': 'mp4',
-            'title': 'E3 2013: Debut Trailer',
-            'description': 'Faith is back!  Check out the World Premiere trailer for Mirror\'s Edge 2 straight from the EA Press Conference at E3 2013!',
+            'display_id': '116437-Just-Cause-3-Review',
+            'title': 'Just Cause 3 - Review',
+            'description': 'It\'s a lot of fun to shoot at things and then watch them explode in Just Cause 3, but should there be more to the experience than that?',
         },
     }
 
-    _FEED_URL = 'http://www.gametrailers.com/feeds/mrss'
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        title = self._html_search_regex(
+            r'<title>(.+?)\|', webpage, 'title').strip()
+        embed_url = self._proto_relative_url(
+            self._search_regex(
+                r'src=\'(//embed.gametrailers.com/embed/[^\']+)\'', webpage,
+                'embed url'),
+            scheme='http:')
+        video_id = url_basename(embed_url)
+        embed_page = self._download_webpage(embed_url, video_id)
+        embed_vars_json = self._search_regex(
+            r'(?s)var embedVars = (\{.*?\})\s*</script>', embed_page,
+            'embed vars')
+        info = self._parse_json(embed_vars_json, video_id)
+
+        formats = []
+        for media in info['media']:
+            if media['mediaPurpose'] == 'play':
+                formats.append({
+                    'url': media['uri'],
+                    'height': media['height'],
+                    'width:': media['width'],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': info.get('thumbUri'),
+            'description': self._og_search_description(webpage),
+            'duration': int_or_none(info.get('videoLengthInSeconds')),
+            'age_limit': parse_age_limit(info.get('audienceRating')),
+        }

From 49358274d7145788f2e99471dbb7c7c4301a3d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 20:49:14 +0600
Subject: [PATCH 2013/2145] [bbc] Fix _VALID_URL

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5f7265a06..7fb80aa38 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -23,7 +23,7 @@ class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _ID_REGEX = r'[pb][\da-z]{7}'
-    _VALID_URL = r'https?://(?:(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])|)(?P<id>%s)' % _ID_REGEX
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>%s)' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails

From 62d231c00493c329a78a30b15a336f131fc392f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 20:55:02 +0600
Subject: [PATCH 2014/2145] [extractor/common] Clarify duration can be float

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eb9bfa3d1..6ab2d68d6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -167,7 +167,7 @@ class InfoExtractor(object):
                     "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
-    duration:       Length of the video in seconds, as an integer.
+    duration:       Length of the video in seconds, as an integer or float.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video

From af93fcfa0549bdfe26c39adecbb1bcc98a9345af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 23:23:36 +0600
Subject: [PATCH 2015/2145] [beeg] Update API URL (Closes #7736)

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 61bc2f744..1ee4a8b05 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -29,7 +29,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v1/video/%s' % video_id, video_id)
+            'http://beeg.com/api/v3/video/%s' % video_id, video_id)
 
         formats = []
         for format_id, video_url in video.items():

From e0977d7686e5df524b1a024484e7a4bb9cfa261d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Dec 2015 00:59:32 +0600
Subject: [PATCH 2016/2145] [beeg] Decrypt URL (Closes #7736)

---
 youtube_dl/extractor/beeg.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 1ee4a8b05..e63c2ac00 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,6 +1,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_chr,
+    compat_ord,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     int_or_none,
     parse_iso8601,
@@ -31,6 +36,23 @@ class BeegIE(InfoExtractor):
         video = self._download_json(
             'http://beeg.com/api/v3/video/%s' % video_id, video_id)
 
+        def decrypt_key(key):
+            # Reverse engineered from http://static.beeg.com/cpl/1067.js
+            a = '8RPUUCS35ZWp3ADnKcSmpH71ZusrROo'
+            e = compat_urllib_parse_unquote(key)
+            return ''.join([
+                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 25)
+                for n in range(len(e))])
+
+        def decrypt_url(encrypted_url):
+            encrypted_url = self._proto_relative_url(
+                encrypted_url.replace('{DATA_MARKERS}', ''), 'http:')
+            key = self._search_regex(
+                r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
+            if not key:
+                return encrypted_url
+            return encrypted_url.replace(key, decrypt_key(key))
+
         formats = []
         for format_id, video_url in video.items():
             if not video_url:
@@ -40,7 +62,7 @@ class BeegIE(InfoExtractor):
             if not height:
                 continue
             formats.append({
-                'url': self._proto_relative_url(video_url.replace('{DATA_MARKERS}', ''), 'http:'),
+                'url': decrypt_url(video_url),
                 'format_id': format_id,
                 'height': int(height),
             })

From 76ab842d9b1308e4c3409cfdfcd925fcdb92e14f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Dec 2015 22:01:32 +0100
Subject: [PATCH 2017/2145] [bilibili] use xpath_text and catch errors in xml
 document

---
 youtube_dl/extractor/bilibili.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 1c3644587..b5aaa41e7 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -7,11 +7,13 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
+    compat_str,
 )
 from ..utils import (
     int_or_none,
     unescapeHTML,
     ExtractorError,
+    xpath_text,
 )
 
 
@@ -58,27 +60,22 @@ class BiliBiliIE(InfoExtractor):
         cid = view_data['cid']
         title = unescapeHTML(view_data['title'])
 
-        page = self._download_webpage(
+        doc = self._download_xml(
             'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
             cid,
             'Downloading page %s/%s' % (page_num, view_data['pages'])
         )
-        try:
-            err_info = json.loads(page)
-            raise ExtractorError(
-                'BiliBili said: ' + err_info['error_text'], expected=True)
-        except ValueError:
-            pass
 
-        doc = compat_etree_fromstring(page)
+        if xpath_text(doc, './result') == 'error':
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, xpath_text(doc, './message')), expected=True)
 
         entries = []
 
         for durl in doc.findall('./durl'):
-            size = durl.find('./filesize|./size')
+            size = xpath_text(durl, ['./filesize', './size'])
             formats = [{
                 'url': durl.find('./url').text,
-                'filesize': int_or_none(size.text) if size else None,
+                'filesize': int_or_none(size),
                 'ext': 'flv',
             }]
             backup_urls = durl.find('./backup_url')
@@ -88,21 +85,21 @@ class BiliBiliIE(InfoExtractor):
             formats.reverse()
 
             entries.append({
-                'id': '%s_part%s' % (cid, durl.find('./order').text),
+                'id': '%s_part%s' % (cid, xpath_text(durl, './order')),
                 'title': title,
-                'duration': int_or_none(durl.find('./length').text) // 1000,
+                'duration': int_or_none(xpath_text(durl, './length'), 1000),
                 'formats': formats,
             })
 
         info = {
-            'id': str(cid),
+            'id': compat_str(cid),
             'title': title,
             'description': view_data.get('description'),
             'thumbnail': view_data.get('pic'),
             'uploader': view_data.get('author'),
             'timestamp': int_or_none(view_data.get('created')),
             'view_count': view_data.get('play'),
-            'duration': int_or_none(doc.find('./timelength').text),
+            'duration': int_or_none(xpath_text(doc, './timelength')),
         }
 
         if len(entries) == 1:

From e5b4225f7c891c4971fe5acdf20459594d5ff9c2 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Dec 2015 22:25:08 +0100
Subject: [PATCH 2018/2145] [audimedia] flake8

---
 youtube_dl/extractor/audimedia.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index cad503510..b0b089dee 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -31,8 +31,8 @@ class AudiMediaIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        
-        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload');
+
+        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload')
         _, stage_mode, video_id, lang = raw_payload.split('-')
 
         # TODO: handle s and e stage_mode (live streams and ended live streams)

From 6612a34939d49d75b2f35d46b1ae9657c11e0f97 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Dec 2015 22:43:19 +0100
Subject: [PATCH 2019/2145] [bilibili] flake8

---
 youtube_dl/extractor/bilibili.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index b5aaa41e7..59beb11bc 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -2,13 +2,9 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_str,
-)
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     unescapeHTML,
@@ -98,7 +94,7 @@ class BiliBiliIE(InfoExtractor):
             'thumbnail': view_data.get('pic'),
             'uploader': view_data.get('author'),
             'timestamp': int_or_none(view_data.get('created')),
-            'view_count': view_data.get('play'),
+            'view_count': int_or_none(view_data.get('play')),
             'duration': int_or_none(xpath_text(doc, './timelength')),
         }
 

From 4ccabf93db1839c07b9fa64e669ce54f33199057 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 00:51:02 +0100
Subject: [PATCH 2020/2145] [trilulilu] fix info extraction

---
 youtube_dl/extractor/trilulilu.py | 118 ++++++++++++++++--------------
 1 file changed, 64 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 185accc4b..3f523ec19 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -1,80 +1,90 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/(?:video-[^/]+/)?(?P<id>[^/#\?]+)'
-    _TEST = {
-        'url': 'http://www.trilulilu.ro/video-animatie/big-buck-bunny-1',
-        'md5': 'c1450a00da251e2769b74b9005601cac',
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?trilulilu\.ro/(?:[^/]+/)?(?P<id>[^/#\?]+)'
+    _TESTS = [{
+        'url': 'http://www.trilulilu.ro/big-buck-bunny-1',
+        'md5': '68da087b676a6196a413549212f60cc6',
         'info_dict': {
             'id': 'ae2899e124140b',
             'ext': 'mp4',
             'title': 'Big Buck Bunny',
             'description': ':) pentru copilul din noi',
+            'uploader_id': 'chipy',
+            'upload_date': '20120304',
+            'timestamp': 1330830647,
+            'uploader': 'chipy',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
         },
-    }
+    }, {
+        'url': 'http://www.trilulilu.ro/adena-ft-morreti-inocenta',
+        'md5': '929dfb8729dc71750463af88bbbbf4a4',
+        'info_dict': {
+            'id': 'f299710e3c91c5',
+            'ext': 'mp4',
+            'title': 'Adena ft. Morreti - Inocenta',
+            'description': 'pop music',
+            'uploader_id': 'VEVOmixt',
+            'upload_date': '20151204',
+            'uploader': 'VEVOmixt',
+            'timestamp': 1449187937,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
+        user = media_info.get('user', {})
 
-        if re.search(r'Fişierul nu este disponibil pentru vizionare în ţara dumneavoastră', webpage):
-            raise ExtractorError(
-                'This video is not available in your country.', expected=True)
-        elif re.search('Fişierul poate fi accesat doar de către prietenii lui', webpage):
-            raise ExtractorError('This video is private.', expected=True)
+        thumbnail = media_info.get('cover_url')
+        if thumbnail:
+            thumbnail.format(width='1600', height='1200')
 
-        flashvars_str = self._search_regex(
-            r'block_flash_vars\s*=\s*(\{[^\}]+\})', webpage, 'flashvars', fatal=False, default=None)
-
-        if flashvars_str:
-            flashvars = self._parse_json(flashvars_str, display_id)
+        media_class = media_info.get('class')
+        if media_class in ('video', 'audio'):
+            # TODO: get correct ext for audio files
+            stream_type = media_info.get('stream_type')
+            formats = [{
+                'url': media_info['href'],
+                'ext': stream_type,
+            }]
+            if media_info.get('is_hd'):
+                formats.append({
+                    'url': media_info['hrefhd'],
+                    'ext': stream_type,
+                })
         else:
-            raise ExtractorError(
-                'This page does not contain videos', expected=True)
+            raise ExtractorError('not a video or an audio')
 
-        if flashvars['isMP3'] == 'true':
-            raise ExtractorError(
-                'Audio downloads are currently not supported', expected=True)
-
-        video_id = flashvars['hash']
-        title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage, default=None)
-
-        format_url = ('http://fs%(server)s.trilulilu.ro/%(hash)s/'
-                      'video-formats2' % flashvars)
-        format_doc = self._download_xml(
-            format_url, video_id,
-            note='Downloading formats',
-            errnote='Error while downloading formats')
-
-        video_url_template = (
-            'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
-            '&source=site&hash=%(hash)s&username=%(userid)s&'
-            'key=ministhebest&format=%%s&sig=&exp=' %
-            flashvars)
-        formats = [
-            {
-                'format_id': fnode.text.partition('-')[2],
-                'url': video_url_template % fnode.text,
-                'ext': fnode.text.partition('-')[0]
-            }
-
-            for fnode in format_doc.findall('./formats/format')
-        ]
+        if media_class == 'audio':
+            formats[0]['vcodec'] = 'none'
 
         return {
-            'id': video_id,
+            'id': media_info['identifier'].split('|')[1],
             'display_id': display_id,
             'formats': formats,
-            'title': title,
-            'description': description,
+            'title': media_info['title'],
+            'description': media_info.get('description'),
             'thumbnail': thumbnail,
+            'uploader_id': user.get('username'),
+            'uploader': user.get('fullname'),
+            'timestamp': parse_iso8601(media_info.get('published'), ' '),
+            'duration': int_or_none(media_info.get('duration')),
+            'view_count': int_or_none(media_info.get('count_views')),
+            'like_count': int_or_none(media_info.get('count_likes')),
+            'comment_count': int_or_none(media_info.get('count_comments')),
         }

From 71d9fe7818b8c766c3b1190017746d8e51db3081 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 06:53:33 +0100
Subject: [PATCH 2021/2145] [trilulilu] improve extraction

---
 youtube_dl/extractor/trilulilu.py | 33 +++++++++++++++++--------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 3f523ec19..515632527 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -48,30 +48,33 @@ class TriluliluIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
+
+        media_class = media_info.get('class')
+        if media_class not in ('video', 'audio'):
+            raise ExtractorError('not a video or an audio')
+
         user = media_info.get('user', {})
 
         thumbnail = media_info.get('cover_url')
         if thumbnail:
             thumbnail.format(width='1600', height='1200')
 
-        media_class = media_info.get('class')
-        if media_class in ('video', 'audio'):
-            # TODO: get correct ext for audio files
-            stream_type = media_info.get('stream_type')
-            formats = [{
-                'url': media_info['href'],
+        # TODO: get correct ext for audio files
+        stream_type = media_info.get('stream_type')
+        formats = [{
+            'url': media_info['href'],
+            'ext': stream_type,
+        }]
+        if media_info.get('is_hd'):
+            formats.append({
+                'format_id': 'hd',
+                'url': media_info['hrefhd'],
                 'ext': stream_type,
-            }]
-            if media_info.get('is_hd'):
-                formats.append({
-                    'url': media_info['hrefhd'],
-                    'ext': stream_type,
-                })
-        else:
-            raise ExtractorError('not a video or an audio')
-
+            })
         if media_class == 'audio':
             formats[0]['vcodec'] = 'none'
+        else:
+            formats[0]['format_id'] = 'sd'
 
         return {
             'id': media_info['identifier'].split('|')[1],

From 46cc1c65a4b5c57555f2549534788ad004c46c03 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 07:09:48 +0100
Subject: [PATCH 2022/2145] [nba] use xpath utils

---
 youtube_dl/extractor/nba.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index a0cc58c12..77a3b49ef 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -6,6 +6,8 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     int_or_none,
+    xpath_text,
+    xpath_attr,
 )
 
 
@@ -26,7 +28,7 @@ class NBAIE(InfoExtractor):
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
-    },{
+    }, {
         'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
@@ -43,11 +45,11 @@ class NBAIE(InfoExtractor):
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
         video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
-        video_id = video_info.find('slug').text
-        title = video_info.find('headline').text
-        description = video_info.find('description').text
-        duration = parse_duration(video_info.find('length').text)
-        timestamp = int_or_none(video_info.find('dateCreated').attrib.get('uts'))
+        video_id = xpath_text(video_info, 'slug')
+        title = xpath_text(video_info, 'headline')
+        description = xpath_text(video_info, 'description')
+        duration = parse_duration(xpath_text(video_info, 'length'))
+        timestamp = int_or_none(xpath_attr(video_info, 'dateCreated', 'uts'))
 
         thumbnails = []
         for image in video_info.find('images'):

From fad2428f47428b75d7728c338ba61407e2b281bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 08:24:04 +0100
Subject: [PATCH 2023/2145] [gameinformer] split long line

---
 youtube_dl/extractor/gameinformer.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
index 8bb764703..25870c131 100644
--- a/youtube_dl/extractor/gameinformer.py
+++ b/youtube_dl/extractor/gameinformer.py
@@ -10,7 +10,6 @@ class GameInformerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
     _TEST = {
         'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
-        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
         'info_dict': {
             'id': '4515472681001',
             'ext': 'm3u8',
@@ -29,7 +28,9 @@ class GameInformerIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         bc_api_url = self._search_regex(r"getVideo\('([^']+)'", webpage, 'brightcove api url')
-        json_data = self._download_json(bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions', display_id)
+        json_data = self._download_json(
+            bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions',
+            display_id)
 
         return {
             'id': compat_str(json_data['id']),

From 2c3b9f35703def516e455448666af0b4c5b31ec9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 08:37:08 +0100
Subject: [PATCH 2024/2145] [adobetv] use a variable for api base url

---
 youtube_dl/extractor/adobetv.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index d0bfafa45..d65826932 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -14,7 +14,11 @@ from ..utils import (
 )
 
 
-class AdobeTVIE(InfoExtractor):
+class AdobeTVBaseIE(InfoExtractor):
+    _API_BASE_URL = 'http://tv.adobe.com/api/v4/'
+
+
+class AdobeTVIE(AdobeTVBaseIE):
     _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
 
     _TEST = {
@@ -38,7 +42,7 @@ class AdobeTVIE(InfoExtractor):
             language = 'en'
 
         video_data = self._download_json(
-            'http://tv.adobe.com/api/v4/episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
+            self._API_BASE_URL + 'episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
             urlname)['data'][0]
 
         formats = [{
@@ -62,7 +66,7 @@ class AdobeTVIE(InfoExtractor):
         }
 
 
-class AdobeTVPlaylistBaseIE(InfoExtractor):
+class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
     def _parse_page_data(self, page_data):
         return [self.url_result(self._get_element_url(element_data)) for element_data in page_data]
 
@@ -97,11 +101,10 @@ class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
             language = 'en'
         query = 'language=%s&show_urlname=%s' % (language, show_urlname)
 
-        show_data = self._download_json(
-            'http://tv.adobe.com/api/v4/show/get/?%s' % query, show_urlname)['data'][0]
+        show_data = self._download_json(self._API_BASE_URL + 'show/get/?%s' % query, show_urlname)['data'][0]
 
         return self.playlist_result(
-            self._extract_playlist_entries('http://tv.adobe.com/api/v4/episode/?%s' % query, show_urlname),
+            self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname),
             str(show_data['id']),
             show_data['show_name'],
             show_data['show_description'])
@@ -130,7 +133,7 @@ class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
             query += '&category_urlname=%s' % category_urlname
 
         return self.playlist_result(
-            self._extract_playlist_entries('http://tv.adobe.com/api/v4/show/?%s' % query, channel_urlname),
+            self._extract_playlist_entries(self._API_BASE_URL + 'show/?%s' % query, channel_urlname),
             channel_urlname)
 
 

From 7079f8ff1fe524a5711957a426066b94a22a605e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 08:44:18 +0100
Subject: [PATCH 2025/2145] [adobetv] use compat_str

---
 youtube_dl/extractor/adobetv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index d65826932..8753ee2cf 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     parse_duration,
     unified_strdate,
@@ -55,7 +56,7 @@ class AdobeTVIE(AdobeTVBaseIE):
         self._sort_formats(formats)
 
         return {
-            'id': str(video_data['id']),
+            'id': compat_str(video_data['id']),
             'title': video_data['title'],
             'description': video_data.get('description'),
             'thumbnail': video_data.get('thumbnail'),
@@ -105,7 +106,7 @@ class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
 
         return self.playlist_result(
             self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname),
-            str(show_data['id']),
+            compat_str(show_data['id']),
             show_data['show_name'],
             show_data['show_description'])
 

From 2a63b0f1109f3f1c1f2d8fd48bfa49b080048c98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Dec 2015 14:26:34 +0100
Subject: [PATCH 2026/2145] [mixcloud] Fix extraction of the audio url (fixes
 #7751)

---
 youtube_dl/extractor/mixcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index d47aeceda..c2b7ed9ab 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -64,7 +64,8 @@ class MixcloudIE(InfoExtractor):
 
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
-        song_url = preview_url.replace('/previews/', '/c/originals/')
+        song_url = re.sub(r'audiocdn(\d+)', r'stream\1', preview_url)
+        song_url = song_url.replace('/previews/', '/c/originals/')
         if not self._check_url(song_url, track_id, 'mp3'):
             song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
             if not self._check_url(song_url, track_id, 'm4a'):

From 55bec9b658ab6a06f7568ccaec4ef212543aa4f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Dec 2015 14:29:37 +0100
Subject: [PATCH 2027/2145] [clipfish] Remove unused import and style fix

---
 youtube_dl/extractor/clipfish.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index d142e326f..7d8eb27a1 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -34,7 +32,7 @@ class ClipfishIE(InfoExtractor):
             'url': video_info['media_videourl_hls'].replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
             'ext': 'mp4',
             'format_id': 'hls',
-        },{
+        }, {
             'url': video_info['media_videourl'],
             'format_id': 'mp4',
             'width': int_or_none(video_info.get('width')),

From b80d4bebf307a18ea0e4991503f91ef601cb2091 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:04:22 +0100
Subject: [PATCH 2028/2145] [nba] fix extraction errors

---
 youtube_dl/extractor/nba.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 77a3b49ef..d8f901b55 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -44,6 +44,8 @@ class NBAIE(InfoExtractor):
 
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
+        if path.startswith('nba/'):
+            path = path[3:]
         video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
         video_id = xpath_text(video_info, 'slug')
         title = xpath_text(video_info, 'headline')
@@ -61,23 +63,27 @@ class NBAIE(InfoExtractor):
             })
 
         formats = []
-        for video_file in video_info.find('files').iter('file'):
+        for video_file in video_info.findall('.//file'):
             video_url = video_file.text
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls'))
+                m3u8_formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             elif video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds'))
+                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
             else:
                 key = video_file.attrib.get('bitrate')
-                width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups()
+                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
                 formats.append({
                     'format_id': key,
                     'url': video_url,
-                    'width': int_or_none(width),
-                    'height': int_or_none(height),
-                    'tbr': int_or_none(bitrate),
+                    'width': int_or_none(mobj.group(1)),
+                    'height': int_or_none(mobj.group(2)),
+                    'tbr': int_or_none(mobj.group(3)),
                 })
         self._sort_formats(formats)
 

From 53e06b25070580b8b0dab7dd54ddff82f3bd2583 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:18:02 +0100
Subject: [PATCH 2029/2145] [ooyala] fix duration scale

---
 youtube_dl/extractor/byutv.py           | 2 +-
 youtube_dl/extractor/generic.py         | 6 +++---
 youtube_dl/extractor/groupon.py         | 2 +-
 youtube_dl/extractor/howcast.py         | 2 +-
 youtube_dl/extractor/ooyala.py          | 8 ++++----
 youtube_dl/extractor/teachingchannel.py | 2 +-
 youtube_dl/extractor/vice.py            | 2 +-
 7 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index ce25816f0..dda98059e 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -17,7 +17,7 @@ class BYUtvIE(InfoExtractor):
             'description': 'md5:e07269172baff037f8e8bf9956bc9747',
             'title': 'Season 5 Episode 5',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 1486486,
+            'duration': 1486.486,
         },
         'params': {
             'skip_download': True,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b60684f98..e3bdff2d8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -339,7 +339,7 @@ class GenericIE(InfoExtractor):
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                 'ext': 'mp4',
                 'title': '2cc213299525360.mov',  # that's what we get
-                'duration': 238231,
+                'duration': 238.231,
             },
             'add_ie': ['Ooyala'],
         },
@@ -351,7 +351,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': '"Steve Jobs: Man in the Machine" trailer',
                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
-                'duration': 135427,
+                'duration': 135.427,
             },
             'params': {
                 'skip_download': True,
@@ -964,7 +964,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
                 'title': 'This is what separates the Excel masters from the wannabes',
-                'duration': 191933,
+                'duration': 191.933,
             },
             'params': {
                 # m3u8 downloads
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 22ff7182f..63c05b6a6 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -19,7 +19,7 @@ class GrouponIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Bikram Yoga Huntington Beach | Orange County',
                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
-                'duration': 44961,
+                'duration': 44.961,
             },
         }],
         'params': {
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 165b9f39e..e8f51e545 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -16,7 +16,7 @@ class HowcastIE(InfoExtractor):
             'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
             'timestamp': 1276081287,
             'upload_date': '20100609',
-            'duration': 56823,
+            'duration': 56.823,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 3b692e903..35067e271 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -24,7 +24,7 @@ class OoyalaBaseIE(InfoExtractor):
             'title': metadata['title'],
             'description': metadata.get('description'),
             'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
-            'duration': int_or_none(metadata.get('duration')),
+            'duration': float_or_none(metadata.get('duration'), 1000),
         }
 
         formats = []
@@ -78,7 +78,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'ext': 'mp4',
                 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
                 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
-                'duration': 853386,
+                'duration': 853.386,
             },
         }, {
             # Only available for ipad
@@ -87,7 +87,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                 'ext': 'mp4',
                 'title': 'Simulation Overview - Levels of Simulation',
-                'duration': 194948,
+                'duration': 194.948,
             },
         },
         {
@@ -99,7 +99,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
                 'ext': 'mp4',
                 'title': 'Divide Tool Path.mp4',
-                'duration': 204405,
+                'duration': 204.405,
             }
         }
     ]
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index 36a6fc679..e0477382c 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -16,7 +16,7 @@ class TeachingChannelIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'A History of Teaming',
             'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
-            'duration': 422255,
+            'duration': 422.255,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 7df87c31c..3db6286e4 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -15,7 +15,7 @@ class ViceIE(InfoExtractor):
                 'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
                 'ext': 'mp4',
                 'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-                'duration': 725983,
+                'duration': 725.983,
             },
             'params': {
                 # Requires ffmpeg (m3u8 manifest)

From 96929dd1e835842cd3c5fad0255a517731f7008d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:23:44 +0100
Subject: [PATCH 2030/2145] [skynewsarabia] fix extractor name

---
 youtube_dl/extractor/skynewsarabia.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index f09fee102..05e1b02ad 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -9,7 +9,7 @@ from ..utils import (
 )
 
 
-class SkyNewArabiaBaseIE(InfoExtractor):
+class SkyNewsArabiaBaseIE(InfoExtractor):
     _IMAGE_BASE_URL = 'http://www.skynewsarabia.com/web/images'
 
     def _call_api(self, path, value):
@@ -40,7 +40,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
         }
 
 
-class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
+class SkyNewsArabiaIE(SkyNewsArabiaBaseIE):
     IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
     _TEST = {
@@ -66,7 +66,7 @@ class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
         return self._extract_video_info(video_data)
 
 
-class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
+class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE):
     IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
     _TESTS = [{

From bee4c5571a18ac52aada80ac7ce9c3e409c18293 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:38:05 +0100
Subject: [PATCH 2031/2145] [clipfish] improve extraction

---
 youtube_dl/extractor/clipfish.py | 34 +++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 7d8eb27a1..3a47f6fa4 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -26,19 +26,29 @@ class ClipfishIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_info = self._download_json('http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id, video_id)['items'][0]
+        video_info = self._download_json(
+            'http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id,
+            video_id)['items'][0]
 
-        formats = [{
-            'url': video_info['media_videourl_hls'].replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
-            'ext': 'mp4',
-            'format_id': 'hls',
-        }, {
-            'url': video_info['media_videourl'],
-            'format_id': 'mp4',
-            'width': int_or_none(video_info.get('width')),
-            'height': int_or_none(video_info.get('height')),
-            'tbr': int_or_none(video_info.get('bitrate')),
-        }]
+        formats = []
+
+        m3u8_url = video_info.get('media_videourl_hls')
+        if m3u8_url:
+            formats.append({
+                'url': m3u8_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+                'ext': 'mp4',
+                'format_id': 'hls',
+            })
+
+        mp4_url = video_info.get('media_videourl')
+        if mp4_url:
+            formats.append({
+                'url': mp4_url,
+                'format_id': 'mp4',
+                'width': int_or_none(video_info.get('width')),
+                'height': int_or_none(video_info.get('height')),
+                'tbr': int_or_none(video_info.get('bitrate')),
+            })
 
         return {
             'id': video_id,

From 64b7e89c0c1f52502ce4fdb7fda4cc1531e578fd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:26:30 +0800
Subject: [PATCH 2032/2145] [srf] Support audios (closes #7760)

---
 youtube_dl/extractor/srf.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 77eec0bc7..16e1bf2d6 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SrfIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
+    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/(?:tv|radio)/[^/]+/(?P<media_type>video|audio)/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'md5': '4cd93523723beff51bb4bee974ee238d',
@@ -35,6 +35,20 @@ class SrfIE(InfoExtractor):
             'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
             'timestamp': 1373493600,
         },
+    }, {
+        'url': 'http://www.srf.ch/play/radio/hoerspielarchiv-srf-musikwelle/audio/saegel-ohni-wind-von-jakob-stebler?id=415bf3d3-6429-4de7-968d-95866e37cfbc',
+        'md5': '',
+        'info_dict': {
+            'id': '415bf3d3-6429-4de7-968d-95866e37cfbc',
+            'display_id': 'saegel-ohni-wind-von-jakob-stebler',
+            'ext': 'mp3',
+            'upload_date': '20080518',
+            'title': '«Sägel ohni Wind» von Jakob Stebler',
+            'timestamp': 1211112000,
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
     }, {
         'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'only_matching': True,
@@ -44,11 +58,13 @@ class SrfIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        display_id = re.match(self._VALID_URL, url).group('display_id') or video_id
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        media_type = mobj.group('media_type')
+        display_id = mobj.group('display_id') or video_id
 
         video_data = self._download_xml(
-            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
+            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/%s/play/%s.xml' % (media_type, video_id),
             display_id)
 
         title = xpath_text(
@@ -64,7 +80,7 @@ class SrfIE(InfoExtractor):
             for url_node in item.findall('url'):
                 quality = url_node.attrib['quality']
                 full_url = url_node.text
-                original_ext = determine_ext(full_url)
+                original_ext = determine_ext(full_url).lower()
                 format_id = '%s-%s' % (quality, item.attrib['protocol'])
                 if original_ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(

From 02e3952f3b2a289dd69ee2f922d7e8f5d8759e4d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 09:29:14 +0100
Subject: [PATCH 2033/2145] [trilulilu] handle errors

---
 youtube_dl/extractor/trilulilu.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 515632527..a800449e9 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -49,6 +49,15 @@ class TriluliluIE(InfoExtractor):
         display_id = self._match_id(url)
         media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
 
+        age_limit = 0
+        errors = media_info.get('errors', {})
+        if errors.get('friends'):
+            raise ExtractorError('This video is private.', expected=True)
+        elif errors.get('geoblock'):
+            raise ExtractorError('This video is not available in your country.', expected=True)
+        elif errors.get('xxx_unlogged'):
+            age_limit = 18
+
         media_class = media_info.get('class')
         if media_class not in ('video', 'audio'):
             raise ExtractorError('not a video or an audio')
@@ -90,4 +99,5 @@ class TriluliluIE(InfoExtractor):
             'view_count': int_or_none(media_info.get('count_views')),
             'like_count': int_or_none(media_info.get('count_likes')),
             'comment_count': int_or_none(media_info.get('count_comments')),
+            'age_limit': age_limit,
         }

From 679bacf0b5bea22944cf95acd1fdeb064b98d7c7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:45:24 +0800
Subject: [PATCH 2034/2145] [bbc.co.uk] Fix test_BBCCoUk

This is similar to the one in #7756, So also fixes #7756.
---
 youtube_dl/extractor/bbc.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7fb80aa38..cd129ba71 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -47,9 +47,8 @@ class BBCCoUkIE(InfoExtractor):
             'info_dict': {
                 'id': 'b039d07m',
                 'ext': 'flv',
-                'title': 'Kaleidoscope, Leonard Cohen',
+                'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4',
                 'description': 'The Canadian poet and songwriter reflects on his musical career.',
-                'duration': 1740,
             },
             'params': {
                 # rtmp download
@@ -454,6 +453,7 @@ class BBCCoUkIE(InfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
         programme_id = None
+        duration = None
 
         tviplayer = self._search_regex(
             r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
@@ -473,7 +473,9 @@ class BBCCoUkIE(InfoExtractor):
             title = self._og_search_title(webpage)
             description = self._search_regex(
                 r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
-                webpage, 'description', fatal=False)
+                webpage, 'description', default=None)
+            if not description:
+                description = self._html_search_meta('description', webpage)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 

From a3ef0e1cdd05698453f30f9250361c42a9fa58df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:51:13 +0800
Subject: [PATCH 2035/2145] [bbc.co.uk] Skip removed test video

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index cd129ba71..d789c3c5b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -111,7 +111,8 @@ class BBCCoUkIE(InfoExtractor):
             'params': {
                 # rtmp download
                 'skip_download': True,
-            }
+            },
+            'skip': 'Episode is no longer available on BBC iPlayer Radio',
         }, {
             'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
             'note': 'Audio',

From 0bc4ee60e09ba8d94cfd4092f373b7b877fa053c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:54:25 +0800
Subject: [PATCH 2036/2145] [bbc] Fix test_BBC_6

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index d789c3c5b..d89e34ba0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -590,6 +590,7 @@ class BBCIE(BBCCoUkIE):
             'ext': 'mp4',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
+            'description': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
         },
         'params': {
             'skip_download': True,

From ab3b773bbecfa4c3a009f141e720a455b1073b61 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 10:14:34 +0100
Subject: [PATCH 2037/2145] [acast] change tests into more stable casts and
 work with channel extractor only if it didn't match cast regex

---
 youtube_dl/extractor/acast.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index c2719ca32..be7913bc7 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -16,15 +16,15 @@ class ACastIE(ACastBaseIE):
     IE_NAME = 'acast'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
     _TEST = {
-        'url': 'https://www.acast.com/gardenersquestiontime/liverpool',
-        'md5': '9e9cd59c3a8a7d8d5407605f51093050',
+        'url': 'https://www.acast.com/condenasttraveler/-where-are-you-taipei-101-taiwan',
+        'md5': 'ada3de5a1e3a2a381327d749854788bb',
         'info_dict': {
-            'id': '43da2262-ade7-420c-8564-f6367da7c010',
+            'id': '57de3baa-4bb0-487e-9418-2692c1277a34',
             'ext': 'mp3',
-            'title': 'Liverpool',
-            'timestamp': 1446163200000,
-            'description': 'md5:170432c9956eec0670d7080a75000d5b',
-            'duration': 2520,
+            'title': '"Where Are You?": Taipei 101, Taiwan',
+            'timestamp': 1196172000000,
+            'description': 'md5:0c5d8201dfea2b93218ea986c91eee6e',
+            'duration': 211,
         }
     }
 
@@ -48,15 +48,19 @@ class ACastChannelIE(ACastBaseIE):
     IE_NAME = 'acast:channel'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
     _TEST = {
-        'url': 'https://www.acast.com/gardenersquestiontime',
+        'url': 'https://www.acast.com/condenasttraveler',
         'info_dict': {
-            'id': '9d8f6f73-6b9d-4d16-9399-52bf88f8f611',
-            'title': 'Gardeners\' Question Time',
-            'description': 'md5:c7ef18049da6a52b63d371b3edccce90',
+            'id': '50544219-29bb-499e-a083-6087f4cb7797',
+            'title': 'Condé Nast Traveler Podcast',
+            'description': 'md5:98646dee22a5b386626ae31866638fbd',
         },
-        'playlist_mincount': 5,
+        'playlist_mincount': 20,
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
         channel_data = self._download_json(self._API_BASE_URL + 'channels/%s' % display_id, display_id)

From db9bd5267f31225fb149b11941bf89818eacf4f7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 17:25:14 +0800
Subject: [PATCH 2038/2145] [keezmovies] Fix extraction

Also fixes #7752
---
 youtube_dl/extractor/keezmovies.py | 38 +++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index d79261bb5..126ca13df 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -1,23 +1,25 @@
 from __future__ import unicode_literals
 
-import os
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
-from ..utils import sanitized_Request
+from ..utils import (
+    sanitized_Request,
+    url_basename,
+)
 
 
 class KeezMoviesIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
-        'md5': '6e297b7e789329923fcf83abb67c9289',
+        'md5': '1c1e75d22ffa53320f45eeb07bc4cdc0',
         'info_dict': {
             'id': '1214711',
             'ext': 'mp4',
             'title': 'Petite Asian Lady Mai Playing In Bathtub',
             'age_limit': 18,
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -36,21 +38,29 @@ class KeezMoviesIE(InfoExtractor):
 
         video_title = self._html_search_regex(
             r'<h1 [^>]*>([^<]+)', webpage, 'title')
-        video_url = self._html_search_regex(
-            r'(?s)html5VideoPlayer = .*?src="([^"]+)"', webpage, 'video URL')
-        path = compat_urllib_parse_urlparse(video_url).path
-        extension = os.path.splitext(path)[1][1:]
-        format = path.split('/')[4].split('_')[:2]
-        format = "-".join(format)
+        flashvars = self._parse_json(self._search_regex(
+            r'var\s+flashvars\s*=\s*([^;]+);', webpage, 'flashvars'), video_id)
+
+        formats = []
+        for height in (180, 240, 480):
+            if flashvars.get('quality_%dp' % height):
+                video_url = flashvars['quality_%dp' % height]
+                a_format = {
+                    'url': video_url,
+                    'height': height,
+                    'format_id': '%dp' % height,
+                }
+                filename_parts = url_basename(video_url).split('_')
+                if len(filename_parts) >= 2 and re.match(r'\d+[Kk]', filename_parts[1]):
+                    a_format['tbr'] = int(filename_parts[1][:-1])
+                formats.append(a_format)
 
         age_limit = self._rta_search(webpage)
 
         return {
             'id': video_id,
             'title': video_title,
-            'url': video_url,
-            'ext': extension,
-            'format': format,
-            'format_id': format,
+            'formats': formats,
             'age_limit': age_limit,
+            'thumbnail': flashvars.get('image_url')
         }

From 38d05d17e54ead57eefd7b33c5ff261f127315a8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 21:10:26 +0800
Subject: [PATCH 2039/2145] [fc2] Fix test_FC2_1

---
 youtube_dl/extractor/fc2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 92e8c571f..4c81271d3 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -37,8 +37,8 @@ class FC2IE(InfoExtractor):
         'params': {
             'username': 'ytdl@yt-dl.org',
             'password': '(snip)',
-            'skip': 'requires actual password'
-        }
+        },
+        'skip': 'requires actual password',
     }, {
         'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
         'only_matching': True,

From ea14422ff12a5184794880befe50f1cac4686ea6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Dec 2015 20:42:21 +0600
Subject: [PATCH 2040/2145] [hypem] Correctly handle cookies (Closes #7762)

---
 youtube_dl/extractor/hypem.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index cca3dd498..c8a74bfc7 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -33,7 +33,6 @@ class HypemIE(InfoExtractor):
         request = sanitized_Request(complete_url)
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
-        cookie = urlh.headers.get('Set-Cookie', '')
 
         html_tracks = self._html_search_regex(
             r'(?ms)<script type="application/json" id="displayList-data">\s*(.*?)\s*</script>',
@@ -52,7 +51,6 @@ class HypemIE(InfoExtractor):
         serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
         request = sanitized_Request(
             serve_url, '', {'Content-Type': 'application/json'})
-        request.add_header('cookie', cookie)
         song_data = self._download_json(request, track_id, 'Downloading metadata')
         final_url = song_data["url"]
 

From 960e0388869a0cb383ee86fa01dce6cc8abc380a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Dec 2015 20:46:57 +0600
Subject: [PATCH 2041/2145] [hypem] Modernize

---
 youtube_dl/extractor/hypem.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index c8a74bfc7..b3706fe6d 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -28,14 +28,12 @@ class HypemIE(InfoExtractor):
         track_id = self._match_id(url)
 
         data = {'ax': 1, 'ts': time.time()}
-        data_encoded = compat_urllib_parse.urlencode(data)
-        complete_url = url + "?" + data_encoded
-        request = sanitized_Request(complete_url)
+        request = sanitized_Request(url + '?' + compat_urllib_parse.urlencode(data))
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
 
         html_tracks = self._html_search_regex(
-            r'(?ms)<script type="application/json" id="displayList-data">\s*(.*?)\s*</script>',
+            r'(?ms)<script type="application/json" id="displayList-data">(.+?)</script>',
             response, 'tracks')
         try:
             track_list = json.loads(html_tracks)
@@ -45,14 +43,14 @@ class HypemIE(InfoExtractor):
 
         key = track['key']
         track_id = track['id']
-        artist = track['artist']
         title = track['song']
 
-        serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
         request = sanitized_Request(
-            serve_url, '', {'Content-Type': 'application/json'})
+            'http://hypem.com/serve/source/%s/%s' % (track_id, key),
+            '', {'Content-Type': 'application/json'})
         song_data = self._download_json(request, track_id, 'Downloading metadata')
-        final_url = song_data["url"]
+        final_url = song_data['url']
+        artist = track.get('artist')
 
         return {
             'id': track_id,

From 535d7b681b05a37a3c2549c7501eb7fe9c8a5f4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 5 Dec 2015 16:01:37 +0100
Subject: [PATCH 2042/2145] release 2015.12.05

---
 README.md              | 2 +-
 docs/supportedsites.md | 8 ++++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index df419abe8..0b224f0be 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,7 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific
                                      one is requested
-    -F, --list-formats               List all available formats of specified
+    -F, --list-formats               List all available formats of requested
                                      videos
     --youtube-skip-dash-manifest     Do not download the DASH manifests and
                                      related data on YouTube videos
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1df408610..880da0e47 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -15,8 +15,12 @@
  - **abc.net.au**
  - **Abc7News**
  - **AcademicEarth:Course**
+ - **acast**
+ - **acast:channel**
  - **AddAnime**
  - **AdobeTV**
+ - **AdobeTVChannel**
+ - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
  - **Aftenposten**
@@ -43,6 +47,7 @@
  - **arte.tv:future**
  - **AtresPlayer**
  - **ATTTechChannel**
+ - **AudiMedia**
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
@@ -183,6 +188,7 @@
  - **freespeech.org**
  - **FreeVideo**
  - **FunnyOrDie**
+ - **GameInformer**
  - **Gamekings**
  - **GameOne**
  - **gameone:playlist**
@@ -480,6 +486,8 @@
  - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
  - **Sina**
+ - **skynewsarabia:video**
+ - **skynewsarabia:video**
  - **Slideshare**
  - **Slutload**
  - **smotri**: Smotri.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bd0de9f53..12ca0241d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.27.1'
+__version__ = '2015.12.05'

From 796db212952605606502fd40aa83007f08524cb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 5 Dec 2015 16:12:02 +0100
Subject: [PATCH 2043/2145] [metacafe] Fix video url extraction (closes #7763)

---
 youtube_dl/extractor/metacafe.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 3c786a36d..5b0b42608 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -154,10 +154,10 @@ class MetacafeIE(InfoExtractor):
         # Extract URL, uploader and title from webpage
         self.report_extraction(video_id)
         video_url = None
-        mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
+        mobj = re.search(r'(?m)&(?:media|video)URL=([^&]+)', webpage)
         if mobj is not None:
             mediaURL = compat_urllib_parse_unquote(mobj.group(1))
-            video_ext = mediaURL[-3:]
+            video_ext = determine_ext(mediaURL)
 
             # Extract gdaKey if available
             mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)

From 77334ccb44c06ea2d9308b334aecc612c2f3240e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 5 Dec 2015 16:12:50 +0100
Subject: [PATCH 2044/2145] [metacafe] Fix age limit extraction

---
 youtube_dl/extractor/metacafe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 5b0b42608..67d6271e1 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -229,7 +229,7 @@ class MetacafeIE(InfoExtractor):
 
         age_limit = (
             18
-            if re.search(r'"contentRating":"restricted"', webpage)
+            if re.search(r'(?:"contentRating":|"rating",)"restricted"', webpage)
             else 0)
 
         if isinstance(video_url, list):

From cb160dd531f328b2c741e4712fae560e1cec3f90 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 18:47:15 +0100
Subject: [PATCH 2045/2145] [nba] handle format info properly

---
 youtube_dl/extractor/nba.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index d8f901b55..5a3eb0833 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -77,14 +77,18 @@ class NBAIE(InfoExtractor):
                     formats.extend(f4m_formats)
             else:
                 key = video_file.attrib.get('bitrate')
-                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
-                formats.append({
+                format_info = {
                     'format_id': key,
                     'url': video_url,
-                    'width': int_or_none(mobj.group(1)),
-                    'height': int_or_none(mobj.group(2)),
-                    'tbr': int_or_none(mobj.group(3)),
-                })
+                }
+                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
+                if mobj:
+                    format_info.update({
+                        'width': int_or_none(mobj.group(1)),
+                        'height': int_or_none(mobj.group(2)),
+                        'tbr': int_or_none(mobj.group(3)),
+                    })
+                formats.append(format_info)
         self._sort_formats(formats)
 
         return {

From 756926ff00b1b0bf5face8116657c99d8ae22172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 01:39:28 +0600
Subject: [PATCH 2046/2145] [pluralsight] Add support for widescreen videos
 (Closes #7766)

---
 youtube_dl/extractor/pluralsight.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index aa7dbcb63..336374bf7 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import re
 import json
 import random
 import collections
@@ -14,6 +15,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    qualities,
     sanitized_Request,
 )
 
@@ -140,15 +142,28 @@ class PluralsightIE(PluralsightBaseIE):
             'low': {'width': 640, 'height': 480},
             'medium': {'width': 848, 'height': 640},
             'high': {'width': 1024, 'height': 768},
+            'high-widescreen': {'width': 1280, 'height': 720},
         }
 
+        QUALITIES_PREFERENCE = ('low', 'medium', 'high', 'high-widescreen',)
+        quality_key = qualities(QUALITIES_PREFERENCE)
+
         AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
 
         ALLOWED_QUALITIES = (
-            AllowedQuality('webm', ('high',)),
-            AllowedQuality('mp4', ('low', 'medium', 'high',)),
+            AllowedQuality('webm', ['high', ]),
+            AllowedQuality('mp4', ['low', 'medium', 'high', ]),
         )
 
+        # Some courses also offer widescreen resolution for high quality (see
+        # https://github.com/rg3/youtube-dl/issues/7766)
+        widescreen = True if re.search(
+            r'courseSupportsWidescreenVideoFormats\s*:\s*true', webpage) else False
+        best_quality = 'high-widescreen' if widescreen else 'high'
+        if widescreen:
+            for allowed_quality in ALLOWED_QUALITIES:
+                allowed_quality.qualities.append(best_quality)
+
         # In order to minimize the number of calls to ViewClip API and reduce
         # the probability of being throttled or banned by Pluralsight we will request
         # only single format until formats listing was explicitly requested.
@@ -164,12 +179,12 @@ class PluralsightIE(PluralsightBaseIE):
                         if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
                             return (AllowedQuality(req_ext, (req_quality, )), )
                 req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
-                return (AllowedQuality(req_ext, ('high', )), )
+                return (AllowedQuality(req_ext, (best_quality, )), )
             allowed_qualities = guess_allowed_qualities()
 
         formats = []
-        for ext, qualities in allowed_qualities:
-            for quality in qualities:
+        for ext, qualities_ in allowed_qualities:
+            for quality in qualities_:
                 f = QUALITIES[quality].copy()
                 clip_post = {
                     'a': author,
@@ -205,6 +220,7 @@ class PluralsightIE(PluralsightBaseIE):
                     'url': clip_url,
                     'ext': ext,
                     'format_id': format_id,
+                    'quality': quality_key(quality),
                 })
                 formats.append(f)
         self._sort_formats(formats)

From edc70f4aaf08009c95cb5e91878461cc7b508f68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 01:40:13 +0600
Subject: [PATCH 2047/2145] [pluralsight] Fix format code split while guessing
 quality

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 336374bf7..55c11b3bf 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -172,7 +172,7 @@ class PluralsightIE(PluralsightBaseIE):
         else:
             def guess_allowed_qualities():
                 req_format = self._downloader.params.get('format') or 'best'
-                req_format_split = req_format.split('-')
+                req_format_split = req_format.split('-', 1)
                 if len(req_format_split) > 1:
                     req_ext, req_quality = req_format_split
                     for allowed_quality in ALLOWED_QUALITIES:

From 0017486dcab67583db7896a9d754b195bcee3bad Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 20:58:44 +0100
Subject: [PATCH 2048/2145] [nba] use int instead of int_or_none

---
 youtube_dl/extractor/nba.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 5a3eb0833..9f99dbf36 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -84,9 +84,9 @@ class NBAIE(InfoExtractor):
                 mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
                 if mobj:
                     format_info.update({
-                        'width': int_or_none(mobj.group(1)),
-                        'height': int_or_none(mobj.group(2)),
-                        'tbr': int_or_none(mobj.group(3)),
+                        'width': int(mobj.group(1)),
+                        'height': int(mobj.group(2)),
+                        'tbr': int(mobj.group(3)),
                     })
                 formats.append(format_info)
         self._sort_formats(formats)

From 51c4fec0d58af709a06bd75338b23eace412362f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 21:04:22 +0100
Subject: [PATCH 2049/2145] [nba] use int_or_none for tbr

---
 youtube_dl/extractor/nba.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 9f99dbf36..7c6b7841d 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -86,7 +86,7 @@ class NBAIE(InfoExtractor):
                     format_info.update({
                         'width': int(mobj.group(1)),
                         'height': int(mobj.group(2)),
-                        'tbr': int(mobj.group(3)),
+                        'tbr': int_or_none(mobj.group(3)),
                     })
                 formats.append(format_info)
         self._sort_formats(formats)

From 33d152b6cc04bfdf658691dbf12c1d976cfa0ba3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:29:41 +0600
Subject: [PATCH 2050/2145] [novamov] Move all novamov based extractors to a
 single place

For easier navigation
---
 youtube_dl/extractor/__init__.py  | 10 +++--
 youtube_dl/extractor/movshare.py  | 27 ------------
 youtube_dl/extractor/novamov.py   | 72 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/nowvideo.py  | 28 ------------
 youtube_dl/extractor/videoweed.py | 26 -----------
 5 files changed, 78 insertions(+), 85 deletions(-)
 delete mode 100644 youtube_dl/extractor/movshare.py
 delete mode 100644 youtube_dl/extractor/nowvideo.py
 delete mode 100644 youtube_dl/extractor/videoweed.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c1dd87550..fefaff35b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -357,7 +357,6 @@ from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviezine import MoviezineIE
-from .movshare import MovShareIE
 from .mtv import (
     MTVIE,
     MTVServicesEmbeddedIE,
@@ -423,7 +422,12 @@ from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
-from .novamov import NovaMovIE
+from .novamov import (
+    NovaMovIE,
+    MovShareIE,
+    NowVideoIE,
+    VideoWeedIE,
+)
 from .nowness import (
     NownessIE,
     NownessPlaylistIE,
@@ -433,7 +437,6 @@ from .nowtv import (
     NowTVIE,
     NowTVListIE,
 )
-from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
@@ -744,7 +747,6 @@ from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
-from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
diff --git a/youtube_dl/extractor/movshare.py b/youtube_dl/extractor/movshare.py
deleted file mode 100644
index 6101063f2..000000000
--- a/youtube_dl/extractor/movshare.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class MovShareIE(NovaMovIE):
-    IE_NAME = 'movshare'
-    IE_DESC = 'MovShare'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'movshare\.(?:net|sx|ag)'}
-
-    _HOST = 'www.movshare.net'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
-    _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'
-
-    _TEST = {
-        'url': 'http://www.movshare.net/video/559e28be54d96',
-        'md5': 'abd31a2132947262c50429e1d16c1bfd',
-        'info_dict': {
-            'id': '559e28be54d96',
-            'ext': 'flv',
-            'title': 'dissapeared image',
-            'description': 'optical illusion  dissapeared image  magic illusion',
-        }
-    }
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 6163e8855..0605c0e0d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -92,3 +92,75 @@ class NovaMovIE(InfoExtractor):
             'title': title,
             'description': description
         }
+
+
+class MovShareIE(NovaMovIE):
+    IE_NAME = 'movshare'
+    IE_DESC = 'MovShare'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'movshare\.(?:net|sx|ag)'}
+
+    _HOST = 'www.movshare.net'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
+    _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'
+
+    _TEST = {
+        'url': 'http://www.movshare.net/video/559e28be54d96',
+        'md5': 'abd31a2132947262c50429e1d16c1bfd',
+        'info_dict': {
+            'id': '559e28be54d96',
+            'ext': 'flv',
+            'title': 'dissapeared image',
+            'description': 'optical illusion  dissapeared image  magic illusion',
+        }
+    }
+
+
+class NowVideoIE(NovaMovIE):
+    IE_NAME = 'nowvideo'
+    IE_DESC = 'NowVideo'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
+
+    _HOST = 'www.nowvideo.to'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
+    _TITLE_REGEX = r'<h4>([^<]+)</h4>'
+    _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
+
+    _TEST = {
+        'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
+        'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
+        'info_dict': {
+            'id': '0mw0yow7b6dxa',
+            'ext': 'flv',
+            'title': 'youtubedl test video _BaW_jenozKc.mp4',
+            'description': 'Description',
+        }
+    }
+
+
+class VideoWeedIE(NovaMovIE):
+    IE_NAME = 'videoweed'
+    IE_DESC = 'VideoWeed'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'videoweed\.(?:es|com)'}
+
+    _HOST = 'www.videoweed.es'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
+
+    _TEST = {
+        'url': 'http://www.videoweed.es/file/b42178afbea14',
+        'md5': 'abd31a2132947262c50429e1d16c1bfd',
+        'info_dict': {
+            'id': 'b42178afbea14',
+            'ext': 'flv',
+            'title': 'optical illusion  dissapeared image magic illusion',
+            'description': ''
+        },
+    }
diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
deleted file mode 100644
index 57ee3d366..000000000
--- a/youtube_dl/extractor/nowvideo.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class NowVideoIE(NovaMovIE):
-    IE_NAME = 'nowvideo'
-    IE_DESC = 'NowVideo'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
-
-    _HOST = 'www.nowvideo.to'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
-    _TITLE_REGEX = r'<h4>([^<]+)</h4>'
-    _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
-
-    _TEST = {
-        'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
-        'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
-        'info_dict': {
-            'id': '0mw0yow7b6dxa',
-            'ext': 'flv',
-            'title': 'youtubedl test video _BaW_jenozKc.mp4',
-            'description': 'Description',
-        }
-    }
diff --git a/youtube_dl/extractor/videoweed.py b/youtube_dl/extractor/videoweed.py
deleted file mode 100644
index ca2e50935..000000000
--- a/youtube_dl/extractor/videoweed.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class VideoWeedIE(NovaMovIE):
-    IE_NAME = 'videoweed'
-    IE_DESC = 'VideoWeed'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'videoweed\.(?:es|com)'}
-
-    _HOST = 'www.videoweed.es'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
-
-    _TEST = {
-        'url': 'http://www.videoweed.es/file/b42178afbea14',
-        'md5': 'abd31a2132947262c50429e1d16c1bfd',
-        'info_dict': {
-            'id': 'b42178afbea14',
-            'ext': 'flv',
-            'title': 'optical illusion  dissapeared image magic illusion',
-            'description': ''
-        },
-    }

From 636aa83ed3fa2b2ef01b6919fca4e4d603b9773a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:37:38 +0600
Subject: [PATCH 2051/2145] [cloudtime] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/novamov.py  | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fefaff35b..ed85d0ee5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -427,6 +427,7 @@ from .novamov import (
     MovShareIE,
     NowVideoIE,
     VideoWeedIE,
+    CloudTimeIE,
 )
 from .nowness import (
     NownessIE,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 0605c0e0d..53d656f9d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -164,3 +164,17 @@ class VideoWeedIE(NovaMovIE):
             'description': ''
         },
     }
+
+
+class CloudTimeIE(NovaMovIE):
+    IE_NAME = 'cloudtime'
+    IE_DESC = 'CloudTime'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'cloudtime\.to'}
+
+    _HOST = 'www.cloudtime.to'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _TITLE_REGEX = r'<div[^>]+class=["\']video_det["\'][^>]*>\s*<strong>([^<]+)</strong>'
+
+    _TEST = None

From 36066dd3ee3f90c3feb136ec4c85a03ece5b8585 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:42:00 +0600
Subject: [PATCH 2052/2145] [movshare] Rename to wholecloud

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/novamov.py  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ed85d0ee5..3db5cd6d9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -424,7 +424,7 @@ from .nosvideo import NosVideoIE
 from .nova import NovaIE
 from .novamov import (
     NovaMovIE,
-    MovShareIE,
+    WholeCloudIE,
     NowVideoIE,
     VideoWeedIE,
     CloudTimeIE,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 53d656f9d..bbbb8fe58 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -94,20 +94,20 @@ class NovaMovIE(InfoExtractor):
         }
 
 
-class MovShareIE(NovaMovIE):
-    IE_NAME = 'movshare'
-    IE_DESC = 'MovShare'
+class WholeCloudIE(NovaMovIE):
+    IE_NAME = 'wholecloud'
+    IE_DESC = 'WholeCloud'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'movshare\.(?:net|sx|ag)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': '(?:wholecloud\.net|movshare\.(?:net|sx|ag))'}
 
-    _HOST = 'www.movshare.net'
+    _HOST = 'www.wholecloud.net'
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
     _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
     _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'
 
     _TEST = {
-        'url': 'http://www.movshare.net/video/559e28be54d96',
+        'url': 'http://www.wholecloud.net/video/559e28be54d96',
         'md5': 'abd31a2132947262c50429e1d16c1bfd',
         'info_dict': {
             'id': '559e28be54d96',

From 7ac40e552181ad43b3e2ad9919a643414d8a8434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:42:20 +0600
Subject: [PATCH 2053/2145] [nowvideo] Update test

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index bbbb8fe58..837c91559 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -132,7 +132,7 @@ class NowVideoIE(NovaMovIE):
     _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
 
     _TEST = {
-        'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
+        'url': 'http://www.nowvideo.to/video/0mw0yow7b6dxa',
         'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
         'info_dict': {
             'id': '0mw0yow7b6dxa',

From 8364b6b0b17f4ea20db9bbd160740801951170ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Dec 2015 16:40:35 +0800
Subject: [PATCH 2054/2145] [iqiyi] Update key

Closes #7772
---
 youtube_dl/extractor/iqiyi.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2df1da3f0..f96e12e69 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,9 +205,8 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-10-22 for Zombie::bite
-        # '7223c67061dbea1259d0ceb44f44b6d62288f4f80c972170de5201d2321060270e05'[2:66][0::2]
-        enc_key = '2c76de15dcb44bd28ff0927d50d31620'
+        # last update at 2015-12-06 for Zombie::bite
+        enc_key = '3719f6a1da83ee0aee3488d8802d7696'[::-1]
         return enc_key
 
     def _real_extract(self, url):

From 7d682f0acb44d72a2c1d0d52fae151f3c273874d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 16:16:59 +0600
Subject: [PATCH 2055/2145] [nowtv] Extend _VALID_URL to support jahr URLs
 (Closes #7755)

---
 youtube_dl/extractor/nowtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 67e34b294..fd107aca2 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -71,7 +71,7 @@ class NowTVBaseIE(InfoExtractor):
 
 
 class NowTVIE(NowTVBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:list/[^/]+/)?(?P<id>[^/]+)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -190,6 +190,9 @@ class NowTVIE(NowTVBaseIE):
     }, {
         'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.de/rtl2/zuhause-im-glueck/jahr/2015/11/eine-erschuetternde-diagnose/player',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 222e11d4ae7d424cd71d9ba87762ce169c0a8c87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 16:41:12 +0600
Subject: [PATCH 2056/2145] [bbc] Add another pattern for playlist.sxml (Closes
 #7743)

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index d89e34ba0..691aecc0d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -733,6 +733,7 @@ class BBCIE(BBCCoUkIE):
         # article with multiple videos embedded with playlist.sxml (e.g.
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
+        playlists.extend(re.findall(r'data-media-id="([^"]+/playlist\.sxml)"', webpage))
         if playlists:
             entries = [
                 self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)

From fd8e559c3a9861121d5d89f9095c8d45aa1d9dcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 23:47:10 +0600
Subject: [PATCH 2057/2145] [beeg] Switch to api v4 (Closes #7774)

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index e63c2ac00..d151d38c9 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,7 +34,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v3/video/%s' % video_id, video_id)
+            'http://beeg.com/api/v4/video/%s' % video_id, video_id)
 
         def decrypt_key(key):
             # Reverse engineered from http://static.beeg.com/cpl/1067.js

From bc92621adea280c93ed7ed86cc8f2bec2c27bb36 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 6 Dec 2015 18:51:25 +0100
Subject: [PATCH 2058/2145] release 2015.12.06

---
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 880da0e47..bf26fecd7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -97,6 +97,7 @@
  - **Clipfish**
  - **cliphunter**
  - **Clipsyndicate**
+ - **cloudtime**: CloudTime
  - **Cloudy**
  - **Clubic**
  - **Clyp**
@@ -313,7 +314,6 @@
  - **MovieClips**
  - **MovieFap**
  - **Moviezine**
- - **movshare**: MovShare
  - **MPORA**
  - **MSNBC**
  - **MTV**
@@ -673,6 +673,7 @@
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
  - **Weibo**
+ - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
  - **WNL**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 12ca0241d..a4e9d7072 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.05'
+__version__ = '2015.12.06'

From db7c9da871def2ab6c8520760512a976ea0fe0e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:02:45 +0600
Subject: [PATCH 2059/2145] [pladform] Add _extract_url routine

---
 youtube_dl/extractor/pladform.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index 551c8c9f0..bc559d1df 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -44,6 +46,13 @@ class PladformIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)"', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 45dad7ba1b28321299f973fc6a42bd7de64481d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:03:21 +0600
Subject: [PATCH 2060/2145] [extractor/generic] Use _extract_url for pladform

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e3bdff2d8..c2e8f9b62 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -54,6 +54,7 @@ from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
+from .pladform import PladformIE
 
 
 class GenericIE(InfoExtractor):
@@ -1741,10 +1742,9 @@ class GenericIE(InfoExtractor):
             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
 
         # Look for Pladform embeds
-        mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://out\.pladform\.ru/player\?.+?)"', webpage)
-        if mobj is not None:
-            return self.url_result(mobj.group('url'), 'Pladform')
+        pladform_url = PladformIE._extract_url(webpage)
+        if pladform_url:
+            return self.url_result(pladform_url)
 
         # Look for Playwire embeds
         mobj = re.search(

From c4737bea1716386e8b3455edfbd783fd415c192d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:03:52 +0600
Subject: [PATCH 2061/2145] [vk] Add support for pladform embeds (Closes #7780)

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d99a42a9f..9632c85fa 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -18,6 +18,7 @@ from ..utils import (
     unified_strdate,
 )
 from .vimeo import VimeoIE
+from .pladform import PladformIE
 
 
 class VKIE(InfoExtractor):
@@ -254,6 +255,10 @@ class VKIE(InfoExtractor):
         if vimeo_url is not None:
             return self.url_result(vimeo_url)
 
+        pladform_url = PladformIE._extract_url(info_page)
+        if pladform_url:
+            return self.url_result(pladform_url)
+
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:

From 291a93bafa066530c19244c4d156422cf41e140a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:04:47 +0600
Subject: [PATCH 2062/2145] [vk] Remove unnecessary message

---
 youtube_dl/extractor/vk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9632c85fa..d719096ff 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -262,7 +262,6 @@ class VKIE(InfoExtractor):
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:
-            self.to_screen('rutube video detected')
             rutube_url = self._proto_relative_url(
                 m_rutube.group(1).replace('\\', ''))
             return self.url_result(rutube_url)

From a5e52a1fd486db46e2115b949da29ec6c03b9bac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:05:54 +0600
Subject: [PATCH 2063/2145] [vk] Add test for pladform embed

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d719096ff..90557fa61 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -165,6 +165,11 @@ class VKIE(InfoExtractor):
             # vk wrapper
             'url': 'http://www.biqle.ru/watch/847655_160197695',
             'only_matching': True,
+        },
+        {
+            # pladform embed
+            'url': 'https://vk.com/video-76116461_171554880',
+            'only_matching': True,
         }
     ]
 

From 0d2d967cc77e445c781bf7bd9a00b75b3bc2236e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:14:45 +0600
Subject: [PATCH 2064/2145] [wimp] Fix extraction (Closes #7784)

---
 youtube_dl/extractor/wimp.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index e4f50e64c..022a3d1b5 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -5,7 +5,7 @@ from .youtube import YoutubeIE
 
 
 class WimpIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)/'
+    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.wimp.com/maruexhausted/',
         'md5': 'ee21217ffd66d058e8b16be340b74883',
@@ -28,18 +28,23 @@ class WimpIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],
-            webpage, 'video URL')
-        if YoutubeIE.suitable(video_url):
-            self.to_screen('Found YouTube video')
+
+        youtube_id = self._search_regex(
+            r"videoId\s*:\s*[\"']([0-9A-Za-z_-]{11})[\"']",
+            webpage, 'video URL', default=None)
+        if youtube_id:
             return {
                 '_type': 'url',
-                'url': video_url,
+                'url': youtube_id,
                 'ie_key': YoutubeIE.ie_key(),
             }
 
+        video_url = self._search_regex(
+            r'<video[^>]+>\s*<source src=(["\'])(?P<url>.+?)\1',
+            webpage, 'video URL', group='url')
+
         return {
             'id': video_id,
             'url': video_url,

From 1775612512aba3c38455e534d423c36c330af35d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:18:00 +0600
Subject: [PATCH 2065/2145] [wimp] Improve video URL regex

---
 youtube_dl/extractor/wimp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 022a3d1b5..041ff6c55 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -42,7 +42,7 @@ class WimpIE(InfoExtractor):
             }
 
         video_url = self._search_regex(
-            r'<video[^>]+>\s*<source src=(["\'])(?P<url>.+?)\1',
+            r'<video[^>]+>\s*<source[^>]+src=(["\'])(?P<url>.+?)\1',
             webpage, 'video URL', group='url')
 
         return {

From 3d24bbfbe433e3a2b2257096264bf4e846f6d889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 23:10:57 +0600
Subject: [PATCH 2066/2145] [YoutubeDL] Check formats for merge to be opposite
 (#7786)

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9a8c7da05..c642a1fbf 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1110,6 +1110,12 @@ class YoutubeDL(object):
                                           'contain the video, try using '
                                           '"-f %s+%s"' % (format_2, format_1))
                         return
+                    # Formats must be opposite (video+audio)
+                    if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
+                        self.report_error(
+                            'Both formats %s and %s are video-only, you must specify "-f video+audio"'
+                            % (format_1, format_2))
+                        return
                     output_ext = (
                         formats_info[0]['ext']
                         if self.params.get('merge_output_format') is None

From 5fd6cd64f9d4b9e7fadbea08fef1f3d131e6f041 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=99=BE=E5=93=A5=E5=93=A5?= <jfk_jin@sina.com>
Date: Tue, 8 Dec 2015 14:12:32 +0800
Subject: [PATCH 2067/2145] [sohu]fix 403 forbidden

---
 youtube_dl/extractor/sohu.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index daf6ad555..ea8fc258d 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -158,6 +158,7 @@ class SohuIE(InfoExtractor):
                         'file': clips_url[i],
                         'new': su[i],
                         'prod': 'flash',
+                        'rb': 1,
                     }
 
                     if cdnId is not None:

From 18e4088fad97de0d79e29e0a49a0df3ce719a441 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:15:45 +0600
Subject: [PATCH 2068/2145] [theplatform] Add support for referer protected
 videos wuth explicit SMIL

---
 youtube_dl/extractor/theplatform.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1555aa77c..0bf6726b5 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -16,11 +16,12 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     ExtractorError,
-    xpath_with_ns,
-    unsmuggle_url,
-    int_or_none,
-    url_basename,
     float_or_none,
+    int_or_none,
+    sanitized_Request,
+    unsmuggle_url,
+    url_basename,
+    xpath_with_ns,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -204,7 +205,12 @@ class ThePlatformIE(ThePlatformBaseIE):
             smil_url = url
         # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
         elif '/guid/' in url:
-            webpage = self._download_webpage(url, video_id)
+            headers = {}
+            source_url = smuggled_data.get('source_url')
+            if source_url:
+                headers['Referer'] = source_url
+            request = sanitized_Request(url, headers=headers)
+            webpage = self._download_webpage(request, video_id)
             smil_url = self._search_regex(
                 r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
                 webpage, 'smil url', group='url')

From b46b65ed370340159e77bf875e99fb93133b6492 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:16:14 +0600
Subject: [PATCH 2069/2145] [nbc] Smuggle referer (Closes #7791)

---
 youtube_dl/extractor/nbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index e683d24c4..7de86a969 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     lowercase_escape,
+    smuggle_url,
     unescapeHTML,
 )
 
@@ -67,7 +68,7 @@ class NBCIE(InfoExtractor):
             webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
-        return self.url_result(theplatform_url)
+        return self.url_result(smuggle_url(theplatform_url, {'source_url': url}))
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From 3348243b7ba94e83dfb9d675fa0e701f97d3e220 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:34:26 +0600
Subject: [PATCH 2070/2145] [README.md] Clarify verbose log requirements

---
 README.md | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0b224f0be..e39f71281 100644
--- a/README.md
+++ b/README.md
@@ -800,7 +800,21 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
-**Please include the full output of youtube-dl when run with `-v`**.
+**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+```
+$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version 2015.12.06
+[debug] Git HEAD: 135392e
+[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+```
+**Do not post screenshots of verbose log only plain text is acceptable.**
 
 The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 

From b09c122373ad3a1e04148aeeac735443fd91e39d Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:16:58 +0600
Subject: [PATCH 2071/2145] [nbc] Add another theplatform pattern

---
 youtube_dl/extractor/nbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7de86a969..4c1eca96f 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -63,6 +63,7 @@ class NBCIE(InfoExtractor):
         theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
             [
                 r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+                r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
                 r'"embedURL"\s*:\s*"([^"]+)"'
             ],
             webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))

From 39fa4cc10775d2d2b8a49d2268b11e34f32b3d2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:56:00 +0600
Subject: [PATCH 2072/2145] [cliphunter] Fix extraction (Closes #7796)

---
 youtube_dl/extractor/cliphunter.py | 43 +++++++++++++-----------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index d46592cc5..2996b6b09 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import int_or_none
 
 
 _translation_table = {
@@ -42,31 +42,26 @@ class CliphunterIE(InfoExtractor):
         video_title = self._search_regex(
             r'mediaTitle = "([^"]+)"', webpage, 'title')
 
-        fmts = {}
-        for fmt in ('mp4', 'flv'):
-            fmt_list = self._parse_json(self._search_regex(
-                r'var %sjson\s*=\s*(\[.*?\]);' % fmt, webpage, '%s formats' % fmt), video_id)
-            for f in fmt_list:
-                fmts[f['fname']] = _decode(f['sUrl'])
-
-        qualities = self._parse_json(self._search_regex(
-            r'var player_btns\s*=\s*(.*?);\n', webpage, 'quality info'), video_id)
+        gexo_files = self._parse_json(
+            self._search_regex(
+                r'var\s+gexoFiles\s*=\s*({.+?});', webpage, 'gexo files'),
+            video_id)
 
         formats = []
-        for fname, url in fmts.items():
-            f = {
-                'url': url,
-            }
-            if fname in qualities:
-                qual = qualities[fname]
-                f.update({
-                    'format_id': '%s_%sp' % (determine_ext(url), qual['h']),
-                    'width': qual['w'],
-                    'height': qual['h'],
-                    'tbr': qual['br'],
-                })
-            formats.append(f)
-
+        for format_id, f in gexo_files.items():
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            fmt = f.get('fmt')
+            height = f.get('h')
+            format_id = '%s_%sp' % (fmt, height) if fmt and height else format_id
+            formats.append({
+                'url': _decode(video_url),
+                'format_id': format_id,
+                'width': int_or_none(f.get('w')),
+                'height': int_or_none(height),
+                'tbr': int_or_none(f.get('br')),
+            })
         self._sort_formats(formats)
 
         thumbnail = self._search_regex(

From 612d83b51df2712327a2643880a80c62e4296d71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 23:28:36 +0600
Subject: [PATCH 2073/2145] [pbs] Extend _VALID_URL

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index b787e2a73..7a4ed23a1 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -18,7 +18,7 @@ class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:video\.pbs|watch\.knpb)\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
@@ -174,6 +174,10 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
+        },
+        {
+            'url': 'http://watch.knpb.org/video/2365616055/',
+            'only_matching': True,
         }
     ]
     _ERRORS = {

From d0c8b279daa83261b5644407ffb4a1fdb3e1b0e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 23:34:43 +0600
Subject: [PATCH 2074/2145] [pbs] Add another coveplayer pattern (Closes #7674)

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 7a4ed23a1..ab3c43cfe 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -208,6 +208,7 @@ class PBSIE(InfoExtractor):
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
                 r'class="coveplayerid">([^<]+)<',                       # coveplayer
+                r'<section[^>]+data-coveid="(\d+)"',                    # coveplayer from http://www.pbs.org/wgbh/frontline/film/real-csi/
                 r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>',  # jwplayer
             ]
 

From e15e2ef7a00362fc1e89d9959241efad46325c54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Dec 2015 01:51:34 +0600
Subject: [PATCH 2075/2145] [pbs] Add support for all member stations (#7674)

---
 youtube_dl/extractor/pbs.py | 477 +++++++++++++++++++++++++++++++++++-
 1 file changed, 475 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ab3c43cfe..decc376a2 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -15,16 +15,489 @@ from ..utils import (
 
 
 class PBSIE(InfoExtractor):
+    _STATIONS = (
+        ('pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # AL, http://aptv.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # AL, http://www.gpb.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AL, http://www.mpbonline.org
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # AL, http://www.wnpt.org
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # AL, http://wfsu.org/
+        ('wsre.org', 'WSRE (WSRE)'),  # AL, http://www.wsre.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # AL, http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # AL, http://pba.org/
+        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # AK, http://alaskapublic.org/kakm
+        ('kuac.org', 'KUAC (KUAC)'),  # AK, http://kuac.org/kuac-tv/
+        ('ktoo.org', '360 North (KTOO)'),  # AK, http://www.ktoo.org/
+        ('azpm.org', 'KUAT 6 (KUAT)'),  # AZ, http://www.azpm.org/
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # AZ, http://www.azpbs.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # AZ, http://www.newmexicopbs.org/
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # AZ, http://vegaspbs.org/
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # AR, http://www.aetn.org/
+        ('ket.org', 'KET (WKLE)'),  # AR, http://www.ket.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # AR, http://www.wkno.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # AR, http://www.lpb.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AR, http://www.mpbonline.org
+        ('oeta.tv', 'OETA (KETA)'),  # AR, http://www.oeta.tv
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # AR, http://www.optv.org/
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # AR, http://www.wsiu.org/
+        ('keet.org', 'KEET TV (KEET)'),  # CA, http://www.keet.org
+        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # CA, http://kixe.org/
+        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # CA, http://www.kpbs.org/
+        ('kqed.org', 'KQED (KQED)'),  # CA, http://www.kqed.org
+        ('kvie.org', 'KVIE Public Television (KVIE)'),  # CA, http://www.kvie.org
+        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # CA, http://www.pbssocal.org/
+        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # CA, http://www.valleypbs.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # CA, http://aptv.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CA, http://cptv.org
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # CA, http://www.azpbs.org
+        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # CA, http://www.knpb.org/
+        ('soptv.org', 'SOPTV (KSYS)'),  # CA, http://www.soptv.org
+        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # CA, http://www.klcs.org
+        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # CA, http://www.krcb.org
+        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # CA, http://kvcr.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # CO, http://www.rmpbs.org
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # CO, http://www.kenw.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # CO, http://www.kued.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # CO, http://www.newmexicopbs.org/
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # CO, http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # CO, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # CO, http://www.kbyutv.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CT, http://cptv.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # CT, http://www.thirteen.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # CT, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # CT, http://www.wgby.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # CT, http://www.njtvonline.org/
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # CT, http://www.ripbs.org/home/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # CT, http://www.wliw.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DE, http://www.mpt.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # DE, http://www.weta.org
+        ('whyy.org', 'WHYY (WHYY)'),  # DE, http://www.whyy.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # DE, http://www.njtvonline.org/
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # DE, http://www.wlvt.org/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # DE, http://www.wliw.org/
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # DC, http://www.weta.org
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DC, http://www.mpt.org
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # DC, http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # DC, http://www.whut.org
+        ('wedu.org', 'WEDU PBS (WEDU)'),  # FL, http://www.wedu.org
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # FL, http://wfsu.org/
+        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # FL, http://www.wgcu.org/
+        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # FL, http://www.wjct.org
+        ('wpbt2.org', 'WPBT2 (WPBT)'),  # FL, http://www.wpbt2.org
+        ('wsre.org', 'WSRE (WSRE)'),  # FL, http://www.wsre.org
+        ('wucftv.org', 'WUCF TV (WUCF)'),  # FL, http://wucftv.org
+        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # FL, http://www.wuft.org
+        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # FL, http://www.wxel.org/home/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # FL, http://aptv.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # FL, http://www.gpb.org/
+        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # FL, http://www.wlrn.org/
+        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # FL, http://wusf.org/
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # FL, http://www.njtvonline.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # GA, http://www.gpb.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # GA, http://aptv.org/
+        ('scetv.org', 'ETV (WRLK)'),  # GA, http://www.scetv.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # GA, http://www.unctv.org/
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # GA, http://wfsu.org/
+        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # GA, http://www.wjct.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # GA, http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # GA, http://pba.org/
+        ('pbsguam.org', 'PBS Guam (KGTF)'),  # GU, http://www.pbsguam.org/
+        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # HI, http://www.pbshawaii.org/
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # ID, http://idahoptv.org
+        ('ksps.org', 'KSPS (KSPS)'),  # ID, http://www.ksps.org/home/
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # ID, http://www.kued.org
+        ('opb.org', 'OPB (KOPB)'),  # ID, http://www.opb.org
+        ('soptv.org', 'SOPTV (KSYS)'),  # ID, http://www.soptv.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # ID, http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # ID, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # ID, http://www.kbyutv.org/
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # ID, http://www.kwsu.org
+        ('illinois.edu', 'WILL-TV (WILL)'),  # IL, http://will.illinois.edu/
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IL, http://www.wsec.tv
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IL, http://www.wsiu.org/
+        ('wttw.com', 'WTTW11 (WTTW)'),  # IL, http://www.wttw.com/
+        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # IL, http://www.wtvp.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IL, http://www.iptv.org/
+        ('ket.org', 'KET (WKLE)'),  # IL, http://www.ket.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # IL, http://www.ninenet.org
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IL, http://wfwa.org/
+        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IL, http://www.wfyi.org
+        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # IL, http://www.mptv.org
+        ('wnin.org', 'WNIN (WNIN)'),  # IL, http://www.wnin.org/
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IL, http://www.wnit.org/
+        ('wpt.org', 'WPT (WPNE)'),  # IL, http://www.wpt.org/
+        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IL, http://wvut.org/
+        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IL, http://www.weiu.net
+        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IL, http://www.wqpt.org
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IL, http://www.wycc.org
+        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IL, http://lakeshorepublicmedia.org/
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IN, http://wfwa.org/
+        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IN, http://www.wfyi.org
+        ('wipb.org', 'WIPB-TV (WIPB)'),  # IN, http://wipb.org
+        ('wnin.org', 'WNIN (WNIN)'),  # IN, http://www.wnin.org/
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IN, http://www.wnit.org/
+        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # IN, http://indianapublicmedia.org/tv/
+        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IN, http://wvut.org/
+        ('cetconnect.org', 'CET  (WCET)'),  # IN, http://www.cetconnect.org
+        ('ket.org', 'KET (WKLE)'),  # IN, http://www.ket.org/
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # IN, http://www.thinktv.org
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # IN, http://wbgu.org
+        ('wgvu.org', 'WGVU TV (WGVU)'),  # IN, http://www.wgvu.org/
+        ('illinois.edu', 'WILL-TV (WILL)'),  # IN, http://will.illinois.edu/
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IN, http://www.wsec.tv
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IN, http://www.wsiu.org/
+        ('wttw.com', 'WTTW11 (WTTW)'),  # IN, http://www.wttw.com/
+        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IN, http://lakeshorepublicmedia.org/
+        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IN, http://www.weiu.net
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IN, http://www.wycc.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IA, http://www.iptv.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # IA, http://netnebraska.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # IA, http://www.pioneer.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # IA, http://www.sdpb.org
+        ('tpt.org', 'TPT (KTCA)'),  # IA, http://www.tpt.org
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IA, http://www.wsec.tv
+        ('wpt.org', 'WPT (WPNE)'),  # IA, http://www.wpt.org/
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # IA, http://www.ksmq.org/
+        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IA, http://www.wqpt.org
+        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # KS, http://www.kpts.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # KS, http://ktwu.org
+        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # KS, http://www.shptv.org
+        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # KS, http://kcpt.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # KS, http://netnebraska.org
+        ('oeta.tv', 'OETA (KETA)'),  # KS, http://www.oeta.tv
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # KS, http://www.optv.org/
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # KS, http://www.rmpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # KS, http://www.cpt12.org/
+        ('ket.org', 'KET (WKLE)'),  # KY, http://www.ket.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # KY, http://www.blueridgepbs.org/
+        ('cetconnect.org', 'CET  (WCET)'),  # KY, http://www.cetconnect.org
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # KY, http://easttennesseepbs.org
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # KY, http://www.wnpt.org
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # KY, http://www.thinktv.org
+        ('wcte.org', 'WCTE-TV (WCTE)'),  # KY, http://www.wcte.org
+        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # KY, http://wljt.org/
+        ('wnin.org', 'WNIN (WNIN)'),  # KY, http://www.wnin.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # KY, http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # KY, http://woub.org/tv/index.php?section=5
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # KY, http://www.wsiu.org/
+        ('wvpublic.org', 'WVPB (WVPB)'),  # KY, http://wvpublic.org/
+        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # KY, http://www.wkyupbs.org
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # LA, http://www.lpb.org/
+        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # LA, http://www.wyes.org
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # LA, http://www.aetn.org/
+        ('kera.org', 'KERA 13 (KERA)'),  # LA, http://www.kera.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # LA, http://www.mpbonline.org
+        ('mpbn.net', 'MPBN (WCBB)'),  # ME, http://www.mpbn.net/
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # ME, http://www.mountainlake.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # ME, http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # ME, http://www.vpt.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # ME, http://wgbh.org
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # MD, http://www.mpt.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # MD, http://www.thirteen.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # MD, http://www.weta.org
+        ('whyy.org', 'WHYY (WHYY)'),  # MD, http://www.whyy.org
+        ('witf.org', 'witf (WITF)'),  # MD, http://www.witf.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # MD, http://www.wqed.org/
+        ('wvpublic.org', 'WVPB (WVPB)'),  # MD, http://wvpublic.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # MD, http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # MD, http://www.whut.org
+        ('wliw.org', 'WLIW21 (WLIW)'),  # MD, http://www.wliw.org/
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # MA, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # MA, http://www.wgby.org
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # MA, http://cptv.org
+        ('nhptv.org', 'NHPTV (WENH)'),  # MA, http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # MA, http://www.vpt.org
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # MA, http://www.wmht.org/home/
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # MA, http://www.ripbs.org/home/
+        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # MI, http://www.deltabroadcasting.org
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MI, http://www.dptv.org/
+        ('wcmu.org', 'CMU Public Television (WCMU)'),  # MI, http://www.wcmu.org
+        ('wgvu.org', 'WGVU TV (WGVU)'),  # MI, http://www.wgvu.org/
+        ('wkar.org', 'WKAR-TV (WKAR)'),  # MI, http://wkar.org/
+        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # MI, http://wnmutv.nmu.edu
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MI, http://www.wdse.org/
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # MI, http://wbgu.org
+        ('wgte.org', 'WGTE TV (WGTE)'),  # MI, http://www.wgte.org
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # MI, http://www.wnit.org/
+        ('wpt.org', 'WPT (WPNE)'),  # MI, http://www.wpt.org/
+        ('wttw.com', 'WTTW11 (WTTW)'),  # MI, http://www.wttw.com/
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # MI, http://www.wycc.org
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # MN, http://www.lakelandptv.org
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MN, http://www.wdse.org/
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # MN, http://www.pioneer.org
+        ('tpt.org', 'TPT (KTCA)'),  # MN, http://www.tpt.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MN, http://www.iptv.org/
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MN, http://www.prairiepublic.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # MN, http://www.ninenet.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # MN, http://www.sdpb.org
+        ('wpt.org', 'WPT (WPNE)'),  # MN, http://www.wpt.org/
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # MN, http://www.ksmq.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # MS, http://www.mpbonline.org
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MS, http://www.aetn.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # MS, http://aptv.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MS, http://www.wkno.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # MS, http://www.lpb.org/
+        ('wsre.org', 'WSRE (WSRE)'),  # MS, http://www.wsre.org
+        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # MS, http://www.wyes.org
+        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # MO, http://kcpt.org/
+        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # MO, http://www.kmos.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # MO, http://www.ninenet.org
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # MO, http://www.optv.org/
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MO, http://www.aetn.org/
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MO, http://www.dptv.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MO, http://www.iptv.org/
+        ('ket.org', 'KET (WKLE)'),  # MO, http://www.ket.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MO, http://www.wkno.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # MO, http://ktwu.org
+        ('netnebraska.org', 'NET1 (KUON)'),  # MO, http://netnebraska.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MO, http://www.rmpbs.org
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # MO, http://www.wsec.tv
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # MO, http://www.wsiu.org/
+        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # MT, http://montanapbs.org
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # MT, http://www.azpbs.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # MT, http://idahoptv.org
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MT, http://www.prairiepublic.org/
+        ('ksps.org', 'KSPS (KSPS)'),  # MT, http://www.ksps.org/home/
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MT, http://www.rmpbs.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # MT, http://www.sdpb.org
+        ('netnebraska.org', 'NET1 (KUON)'),  # NE, http://netnebraska.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # NE, http://www.iptv.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # NE, http://ktwu.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # NE, http://www.pioneer.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NE, http://www.rmpbs.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # NE, http://www.sdpb.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # NE, http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NE, http://www.cpt12.org/
+        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # NV, http://www.knpb.org/
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # NV, http://vegaspbs.org/
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # NV, http://www.azpbs.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # NV, http://www.kued.org
+        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # NV, http://www.pbssocal.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NV, http://www.kbyutv.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # NH, http://nhptv.org/
+        ('scetv.org', 'ETV (WRLK)'),  # NH, http://www.scetv.org
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NH, http://www.mountainlake.org/
+        ('mpbn.net', 'MPBN (WCBB)'),  # NH, http://www.mpbn.net/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # NH, http://www.vpt.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NH, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # NH, http://www.wgby.org
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NJ, http://cptv.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NJ, http://www.thirteen.org
+        ('whyy.org', 'WHYY (WHYY)'),  # NJ, http://www.whyy.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NJ, http://www.njtvonline.org/
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # NJ, http://www.wlvt.org/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # NJ, http://www.wliw.org/
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # NM, http://www.kenw.org
+        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # NM, http://www.krwg.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # NM, http://www.newmexicopbs.org/
+        ('azpm.org', 'KUAT 6 (KUAT)'),  # NM, http://www.azpm.org/
+        ('panhandlepbs.org', 'KACV (KACV)'),  # NM, http://www.panhandlepbs.org/home/
+        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # NM, www.kcostv.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # NM, http://www.kued.org
+        ('oeta.tv', 'OETA (KETA)'),  # NM, http://www.oeta.tv
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NM, http://www.rmpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NM, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NM, http://www.kbyutv.org/
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NY, http://www.mountainlake.org/
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NY, http://www.thirteen.org
+        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # NY, http://www.wcny.org
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # NY, http://www.wmht.org/home/
+        ('wned.org', 'WNED (WNED)'),  # NY, http://www.wned.org/
+        ('wpbstv.org', 'WPBS (WPBS)'),  # NY, http://www.wpbstv.org
+        ('wskg.org', 'WSKG Public TV (WSKG)'),  # NY, http://wskg.org
+        ('wxxi.org', 'WXXI (WXXI)'),  # NY, http://wxxi.org
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NY, http://cptv.org
+        ('nhptv.org', 'NHPTV (WENH)'),  # NY, http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # NY, http://www.vpt.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NY, http://wgbh.org
+        ('wpsu.org', 'WPSU (WPSU)'),  # NY, http://www.wpsu.org
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # NY, http://www.wqln.org
+        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # NY, http://www.wvia.org/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # NY, http://www.wliw.org/
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NY, http://www.njtvonline.org/
+        ('unctv.org', 'UNC-TV (WUNC)'),  # NC, http://www.unctv.org/
+        ('wtvi.org', 'WTVI (WTVI)'),  # NC, http://www.wtvi.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # NC, http://www.blueridgepbs.org/
+        ('scetv.org', 'ETV (WRLK)'),  # NC, http://www.scetv.org
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # NC, http://www.gpb.org/
+        ('whro.org', 'WHRO (WHRO)'),  # NC, http://whro.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # NC, http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # NC, http://pba.org/
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # ND, http://www.prairiepublic.org/
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # ND, http://www.sdpb.org
+        ('cetconnect.org', 'CET  (WCET)'),  # OH, http://www.cetconnect.org
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # OH, http://www.thinktv.org
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # OH, http://wbgu.org
+        ('wgte.org', 'WGTE TV (WGTE)'),  # OH, http://www.wgte.org
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # OH, http://www.WesternReservePublicMedia.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # OH, http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # OH, http://woub.org/tv/index.php?section=5
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # OH, http://www.wviz.org/
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # OH, http://www.dptv.org/
+        ('ket.org', 'KET (WKLE)'),  # OH, http://www.ket.org/
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # OH, http://wfwa.org/
+        ('wipb.org', 'WIPB-TV (WIPB)'),  # OH, http://wipb.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # OH, http://www.wqed.org/
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # OH, http://www.wqln.org
+        ('wvpublic.org', 'WVPB (WVPB)'),  # OH, http://wvpublic.org/
+        ('oeta.tv', 'OETA (KETA)'),  # OK, http://www.oeta.tv
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # OK, http://www.aetn.org/
+        ('panhandlepbs.org', 'KACV (KACV)'),  # OK, http://www.panhandlepbs.org/home/
+        ('kera.org', 'KERA 13 (KERA)'),  # OK, http://www.kera.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # OK, http://www.lpb.org/
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # OK, http://www.optv.org/
+        ('opb.org', 'OPB (KOPB)'),  # OR, http://www.opb.org
+        ('soptv.org', 'SOPTV (KSYS)'),  # OR, http://www.soptv.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # OR, http://idahoptv.org
+        ('kcts9.org', 'KCTS 9 (KCTS)'),  # OR, http://kcts9.org/
+        ('keet.org', 'KEET TV (KEET)'),  # OR, http://www.keet.org
+        ('ksps.org', 'KSPS (KSPS)'),  # OR, http://www.ksps.org/home/
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # OR, http://www.kwsu.org
+        ('whyy.org', 'WHYY (WHYY)'),  # PA, http://www.whyy.org
+        ('witf.org', 'witf (WITF)'),  # PA, http://www.witf.org
+        ('wpsu.org', 'WPSU (WPSU)'),  # PA, http://www.wpsu.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # PA, http://www.wqed.org/
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # PA, http://www.wqln.org
+        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # PA, http://www.wvia.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # PA, http://cptv.org
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # PA, http://www.mpt.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # PA, http://www.thirteen.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # PA, http://www.weta.org
+        ('wned.org', 'WNED (WNED)'),  # PA, http://www.wned.org/
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # PA, http://www.WesternReservePublicMedia.org/
+        ('wskg.org', 'WSKG Public TV (WSKG)'),  # PA, http://wskg.org
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # PA, http://www.wviz.org/
+        ('wvpublic.org', 'WVPB (WVPB)'),  # PA, http://wvpublic.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # PA, http://www.wvpt.net
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # PA, http://www.wlvt.org/
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # PA, http://www.njtvonline.org/
+        ('whut.org', 'Howard University Television (WHUT)'),  # PA, http://www.whut.org
+        ('wliw.org', 'WLIW21 (WLIW)'),  # PA, http://www.wliw.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # RI, http://cptv.org
+        ('nhptv.org', 'NHPTV (WENH)'),  # RI, http://nhptv.org/
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # RI, http://wgbh.org
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # RI, http://www.ripbs.org/home/
+        ('scetv.org', 'ETV (WRLK)'),  # SC, http://www.scetv.org
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # SC, http://www.gpb.org/
+        ('unctv.org', 'UNC-TV (WUNC)'),  # SC, http://www.unctv.org/
+        ('wtvi.org', 'WTVI (WTVI)'),  # SC, http://www.wtvi.org/
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # SC, http://pba.org/
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # SD, http://www.sdpb.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # SD, http://www.iptv.org/
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # SD, http://www.prairiepublic.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # SD, http://netnebraska.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # SD, http://www.pioneer.org
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # TN, http://easttennesseepbs.org
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # TN, http://www.wkno.org/
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TN, http://www.wnpt.org
+        ('wcte.org', 'WCTE-TV (WCTE)'),  # TN, http://www.wcte.org
+        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # TN, http://wljt.org/
+        ('wtcitv.org', 'WTCI (WTCI)'),  # TN, http://www.wtcitv.org
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # TN, http://aptv.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # TN, http://www.blueridgepbs.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # TN, http://www.gpb.org/
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # TN, http://www.lakelandptv.org
+        ('ket.org', 'KET (WKLE)'),  # TN, http://www.ket.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # TN, http://www.mpbonline.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # TN, http://www.unctv.org/
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # TN, http://www.WesternReservePublicMedia.org/
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # TN, http://www.wsiu.org/
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # TN, http://www.wviz.org/
+        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # TN, http://www.wkyupbs.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # TN, http://pba.org/
+        ('basinpbs.org', 'Basin PBS (KPBT)'),  # TX, http://www.basinpbs.org
+        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # TX, http://www.houstonpublicmedia.org/
+        ('panhandlepbs.org', 'KACV (KACV)'),  # TX, http://www.panhandlepbs.org/home/
+        ('tamu.edu', 'KAMU - TV (KAMU)'),  # TX, http://KAMU.tamu.edu
+        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # TX, www.kcostv.org
+        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # TX, http://www.kedt.org
+        ('kera.org', 'KERA 13 (KERA)'),  # TX, http://www.kera.org/
+        ('klrn.org', 'KLRN (KLRN)'),  # TX, http://www.klrn.org
+        ('klru.org', 'KLRU (KLRU)'),  # TX, http://www.klru.org
+        ('kmbh.org', 'KMBH-TV (KMBH)'),  # TX, http://www.kmbh.org
+        ('knct.org', 'KNCT (KNCT)'),  # TX, http://www.knct.org
+        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # TX, http://www.ktxt.org
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # TX, http://www.aetn.org/
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # TX, http://www.kenw.org
+        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # TX, http://www.krwg.org
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # TX, http://www.lpb.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # TX, http://netnebraska.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # TX, http://www.newmexicopbs.org/
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TX, http://www.wnpt.org
+        ('oeta.tv', 'OETA (KETA)'),  # TX, http://www.oeta.tv
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # TX, http://www.rmpbs.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # UT, http://www.kued.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # UT, http://idahoptv.org
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # UT, http://vegaspbs.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # UT, http://www.kbyutv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # VT, http://www.vpt.org
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # VT, http://www.mountainlake.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # VT, http://nhptv.org/
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # VT, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # VT, http://www.wgby.org
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # VT, http://www.wmht.org/home/
+        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # VI, http://www.wtjx.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # VA, http://www.blueridgepbs.org/
+        ('ideastations.org', 'WCVE PBS (WCVE)'),  # VA, http://ideastations.org/
+        ('whro.org', 'WHRO (WHRO)'),  # VA, http://whro.org
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # VA, http://www.wvpt.net
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # VA, http://easttennesseepbs.org
+        ('ket.org', 'KET (WKLE)'),  # VA, http://www.ket.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # VA, http://www.mpt.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # VA, http://www.unctv.org/
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # VA, http://www.weta.org
+        ('whut.org', 'Howard University Television (WHUT)'),  # VA, http://www.whut.org
+        ('kcts9.org', 'KCTS 9 (KCTS)'),  # WA, http://kcts9.org/
+        ('ksps.org', 'KSPS (KSPS)'),  # WA, http://www.ksps.org/home/
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WA, http://idahoptv.org
+        ('opb.org', 'OPB (KOPB)'),  # WA, http://www.opb.org
+        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # WA, http://kbtc.org
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # WA, http://www.kwsu.org
+        ('wvpublic.org', 'WVPB (WVPB)'),  # WV, http://wvpublic.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # WV, http://www.blueridgepbs.org/
+        ('ket.org', 'KET (WKLE)'),  # WV, http://www.ket.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # WV, http://www.mpt.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # WV, http://www.weta.org
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # WV, http://www.WesternReservePublicMedia.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # WV, http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # WV, http://woub.org/tv/index.php?section=5
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # WV, http://www.wqed.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # WV, http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # WV, http://www.whut.org
+        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # WI, http://www.mptv.org
+        ('wpt.org', 'WPT (WPNE)'),  # WI, http://www.wpt.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # WI, http://www.iptv.org/
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # WI, http://www.lakelandptv.org
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # WI, http://www.wdse.org/
+        ('tpt.org', 'TPT (KTCA)'),  # WI, http://www.tpt.org
+        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # WI, http://wnmutv.nmu.edu
+        ('wttw.com', 'WTTW11 (WTTW)'),  # WI, http://www.wttw.com/
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # WI, http://www.ksmq.org/
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # WI, http://www.wycc.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # WY, http://www.wyomingpbs.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WY, http://idahoptv.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # WY, http://www.kued.org
+        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # WY, http://montanapbs.org
+        ('netnebraska.org', 'NET1 (KUON)'),  # WY, http://netnebraska.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # WY, http://www.rmpbs.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # WY, http://www.sdpb.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # WY, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # WY, http://www.kbyutv.org/
+    )
+
+    IE_NAME = 'pbs'
+    IE_DESC = 'Public Broadcasting Service (PBS) and member stations: %s' % ', '.join(list(zip(*_STATIONS))[1])
+
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           (?:video\.pbs|watch\.knpb)\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:videos?|watch)\.(?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
-    '''
+    ''' % '|'.join(list(zip(*_STATIONS))[0])
 
     _TESTS = [
         {

From 47f48f5d851cf2f0acd22ad3fb00a2b69378c7bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 8 Dec 2015 21:12:13 +0100
Subject: [PATCH 2076/2145] [test/test_all_urls] Update pbs extractor name

It's in lowercase now (since e15e2ef7a00362fc1e89d9959241efad46325c54).
---
 test/test_all_urls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a9db42b30..a0c11e6c1 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -121,8 +121,8 @@ class TestAllURLsMatching(unittest.TestCase):
 
     def test_pbs(self):
         # https://github.com/rg3/youtube-dl/issues/2350
-        self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['PBS'])
-        self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['PBS'])
+        self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['pbs'])
+        self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['pbs'])
 
     def test_yahoo_https(self):
         # https://github.com/rg3/youtube-dl/issues/2701

From 86e8c89488c4d41009aa8f1044681c3f217ec1b8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 9 Dec 2015 15:32:26 +0100
Subject: [PATCH 2077/2145] release 2015.12.09

---
 CONTRIBUTING.md        | 16 +++++++++++++++-
 docs/supportedsites.md |  2 +-
 youtube_dl/version.py  |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 09ce98ca2..28aae94f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,4 +1,18 @@
-**Please include the full output of youtube-dl when run with `-v`**.
+**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+```
+$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version 2015.12.06
+[debug] Git HEAD: 135392e
+[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+```
+**Do not post screenshots of verbose log only plain text is acceptable.**
 
 The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index bf26fecd7..4f19cb6f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -400,7 +400,7 @@
  - **orf:tvthek**: ORF TVthek
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
- - **PBS**
+ - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), KUAC (KUAC), 360 North (KTOO), KUAT 6 (KUAT), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Mississippi Public Broadcasting (WMPN), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), APT - Alabama Public Television (WBIQ), CONNECTICUT PUBLIC TELEVISION (WEDH), Arizona PBS (KAET), KNPB Channel 5 (KNPB), SOPTV (KSYS), KLCS/Channel 58 (KLCS), KRCB Television & Radio (KRCB), KVCR TV/DT/FM :: Vision for the Future (KVCR), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), KNME-TV/Channel 5 (KNME), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), Rhode Island PBS (WSBE), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), WETA Television and Radio (WETA), mpt/Maryland Public Television (WMPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WFSU-TV (WFSU), WGCU Public Media (WGCU), WJCT Public Broadcasting (WJCT), WPBT2 (WPBT), WSRE (WSRE), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), NJTV Public Media NJ (WNJT), GPB/Georgia Public Broadcasting (WGTV), APT - Alabama Public Television (WBIQ), ETV (WRLK), UNC-TV (WUNC), WFSU-TV (WFSU), WJCT Public Broadcasting (WJCT), WTCI (WTCI), WPBA/Channel 30 (WPBA), PBS Guam (KGTF), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), KUED Channel 7 (KUED), OPB (KOPB), SOPTV (KSYS), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), WTVP & WTVP.org, Public Media for Central Illinois (WTVP), Iowa Public Television/IPTV (KDIN), KET (WKLE), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), Lakeshore Public Television (WYIN), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), WIPB-TV (WIPB), WNIN (WNIN), WNIT Public Television (WNIT), WTIU (WTIU), WVUT/Channel 22 (WVUT), CET  (WCET), KET (WKLE), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), Lakeshore Public Television (WYIN), WEIU/Channel 51 (WEIU), WYCC PBS Chicago (WYCC), Iowa Public Television/IPTV (KDIN), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), Network Knowledge - WSEC/Springfield (WSEC), WPT (WPNE), KSMQ (KSMQ), WQPT-TV (WQPT), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), Smoky Hills Public Television (KOOD), KCPT Kansas City Public Television (KCPT), NET1 (KUON), OETA (KETA), Ozarks Public Television (KOZK), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KET (WKLE), Blue Ridge PBS (WBRA), CET  (WCET), East Tennessee PBS (WSJK), Nashville Public Television (WNPT), ThinkTVNetwork (WPTD), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WNIN (WNIN), WOSU TV (WOSU), WOUB/WOUC (WOUB), WSIU Public Broadcasting (WSIU), WVPB (WVPB), WKYU-PBS (WKYU), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WYES-TV/New Orleans (WYES), AETN/ARKANSAS ETV NETWORK (KETS), KERA 13 (KERA), Mississippi Public Broadcasting (WMPN), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WHYY (WHYY), witf (WITF), WQED Multimedia (WQED), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WLIW21 (WLIW), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WMHT Educational Telecommunications (WMHT), Rhode Island PBS (WSBE), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WGVU TV (WGVU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WBGU-TV (WBGU), WGTE TV (WGTE), WNIT Public Television (WNIT), WPT (WPNE), WTTW11 (WTTW), WYCC PBS Chicago (WYCC), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), Pioneer Public Television (KWCM), TPT (KTCA), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), Nine Network (KETC), SDPB Television (KUSD), WPT (WPNE), KSMQ (KSMQ), Mississippi Public Broadcasting (WMPN), AETN/ARKANSAS ETV NETWORK (KETS), APT - Alabama Public Television (WBIQ), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WSRE (WSRE), WYES-TV/New Orleans (WYES), KCPT Kansas City Public Television (KCPT), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), Nine Network (KETC), Ozarks Public Television (KOZK), AETN/ARKANSAS ETV NETWORK (KETS), WTVS Detroit Public TV (WTVS), Iowa Public Television/IPTV (KDIN), KET (WKLE), WKNO/Channel 10 (WKNO), KTWU/Channel 11 (KTWU), NET1 (KUON), Rocky Mountain PBS (KRMA), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), MontanaPBS (KUSM), Arizona PBS (KAET), Idaho Public Television (KAID), PRAIRIE PUBLIC (KFME), KSPS (KSPS), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), NET1 (KUON), Iowa Public Television/IPTV (KDIN), KTWU/Channel 11 (KTWU), Pioneer Public Television (KWCM), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KNPB Channel 5 (KNPB), Vegas PBS (KLVX), Arizona PBS (KAET), KUED Channel 7 (KUED), PBS SoCal/KOCE (KOCE), KBYU-TV (KBYU), NHPTV (WENH), ETV (WRLK), Mountain Lake PBS (WCFE), MPBN (WCBB), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), KNME-TV/Channel 5 (KNME), KUAT 6 (KUAT), KACV (KACV), KCOS/Channel 13 (KCOS), KUED Channel 7 (KUED), OETA (KETA), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Mountain Lake PBS (WCFE), Thirteen/WNET New York (WNET), WCNY/Channel 24 (WCNY), WMHT Educational Telecommunications (WMHT), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WPSU (WPSU), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), WLIW21 (WLIW), NJTV Public Media NJ (WNJT), UNC-TV (WUNC), WTVI (WTVI), Blue Ridge PBS (WBRA), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), WHRO (WHRO), WTCI (WTCI), WPBA/Channel 30 (WPBA), PRAIRIE PUBLIC (KFME), SDPB Television (KUSD), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGTE TV (WGTE), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVIZ/PBS ideastream (WVIZ), WTVS Detroit Public TV (WTVS), KET (WKLE), PBS39 Fort Wayne (WFWA), WIPB-TV (WIPB), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVPB (WVPB), OETA (KETA), AETN/ARKANSAS ETV NETWORK (KETS), KACV (KACV), KERA 13 (KERA), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Ozarks Public Television (KOZK), OPB (KOPB), SOPTV (KSYS), Idaho Public Television (KAID), KCTS 9 (KCTS), KEET TV (KEET), KSPS (KSPS), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WHYY (WHYY), witf (WITF), WPSU (WPSU), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), CONNECTICUT PUBLIC TELEVISION (WEDH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WNED (WNED), Western Reserve PBS (WNEO), WSKG Public TV (WSKG), WVIZ/PBS ideastream (WVIZ), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), PBS 39 (WLVT), NJTV Public Media NJ (WNJT), Howard University Television (WHUT), WLIW21 (WLIW), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), WGBH/Channel 2 (WGBH), Rhode Island PBS (WSBE), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), UNC-TV (WUNC), WTVI (WTVI), WPBA/Channel 30 (WPBA), SDPB Television (KUSD), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), NET1 (KUON), Pioneer Public Television (KWCM), East Tennessee PBS (WSJK), WKNO/Channel 10 (WKNO), Nashville Public Television (WNPT), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WTCI (WTCI), APT - Alabama Public Television (WBIQ), Blue Ridge PBS (WBRA), GPB/Georgia Public Broadcasting (WGTV), Lakeland Public Television (KAWE), KET (WKLE), Mississippi Public Broadcasting (WMPN), UNC-TV (WUNC), Western Reserve PBS (WNEO), WSIU Public Broadcasting (WSIU), WVIZ/PBS ideastream (WVIZ), WKYU-PBS (WKYU), WPBA/Channel 30 (WPBA), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KACV (KACV), KAMU - TV (KAMU), KCOS/Channel 13 (KCOS), KEDT/Channel 16 (KEDT), KERA 13 (KERA), KLRN (KLRN), KLRU (KLRU), KMBH-TV (KMBH), KNCT (KNCT), KTTZ-TV (KTXT), AETN/ARKANSAS ETV NETWORK (KETS), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), NET1 (KUON), KNME-TV/Channel 5 (KNME), Nashville Public Television (WNPT), OETA (KETA), Rocky Mountain PBS (KRMA), KUED Channel 7 (KUED), Idaho Public Television (KAID), Vegas PBS (KLVX), KBYU-TV (KBYU), Vermont PBS (WETK), Mountain Lake PBS (WCFE), NHPTV (WENH), WGBH/Channel 2 (WGBH), WGBY (WGBY), WMHT Educational Telecommunications (WMHT), WTJX Channel 12 (WTJX), Blue Ridge PBS (WBRA), WCVE PBS (WCVE), WHRO (WHRO), WVPT - Your Source for PBS and More! (WVPT), East Tennessee PBS (WSJK), KET (WKLE), mpt/Maryland Public Television (WMPB), UNC-TV (WUNC), WETA Television and Radio (WETA), Howard University Television (WHUT), KCTS 9 (KCTS), KSPS (KSPS), Idaho Public Television (KAID), OPB (KOPB), KBTC Public Television (KBTC), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WVPB (WVPB), Blue Ridge PBS (WBRA), KET (WKLE), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WQED Multimedia (WQED), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), Milwaukee Public Television (WMVS), WPT (WPNE), Iowa Public Television/IPTV (KDIN), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), TPT (KTCA), WNMU-TV Public TV 13 (WNMU), WTTW11 (WTTW), KSMQ (KSMQ), WYCC PBS Chicago (WYCC), Wyoming PBS (KCWC), Idaho Public Television (KAID), KUED Channel 7 (KUED), MontanaPBS (KUSM), NET1 (KUON), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU)
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a4e9d7072..0f5571006 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.06'
+__version__ = '2015.12.09'

From b51b10804556659af181541e6cd0d46347a8859f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Dec 2015 21:23:19 +0600
Subject: [PATCH 2078/2145] [pbs] Clean up stations list from duplicates

---
 youtube_dl/extractor/pbs.py | 625 +++++++++---------------------------
 1 file changed, 158 insertions(+), 467 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index decc376a2..6aae7f028 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,473 +16,164 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        ('pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # AL, http://aptv.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # AL, http://www.gpb.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AL, http://www.mpbonline.org
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # AL, http://www.wnpt.org
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # AL, http://wfsu.org/
-        ('wsre.org', 'WSRE (WSRE)'),  # AL, http://www.wsre.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # AL, http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # AL, http://pba.org/
-        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # AK, http://alaskapublic.org/kakm
-        ('kuac.org', 'KUAC (KUAC)'),  # AK, http://kuac.org/kuac-tv/
-        ('ktoo.org', '360 North (KTOO)'),  # AK, http://www.ktoo.org/
-        ('azpm.org', 'KUAT 6 (KUAT)'),  # AZ, http://www.azpm.org/
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # AZ, http://www.azpbs.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # AZ, http://www.newmexicopbs.org/
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # AZ, http://vegaspbs.org/
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # AR, http://www.aetn.org/
-        ('ket.org', 'KET (WKLE)'),  # AR, http://www.ket.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # AR, http://www.wkno.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # AR, http://www.lpb.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AR, http://www.mpbonline.org
-        ('oeta.tv', 'OETA (KETA)'),  # AR, http://www.oeta.tv
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # AR, http://www.optv.org/
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # AR, http://www.wsiu.org/
-        ('keet.org', 'KEET TV (KEET)'),  # CA, http://www.keet.org
-        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # CA, http://kixe.org/
-        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # CA, http://www.kpbs.org/
-        ('kqed.org', 'KQED (KQED)'),  # CA, http://www.kqed.org
-        ('kvie.org', 'KVIE Public Television (KVIE)'),  # CA, http://www.kvie.org
-        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # CA, http://www.pbssocal.org/
-        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # CA, http://www.valleypbs.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # CA, http://aptv.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CA, http://cptv.org
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # CA, http://www.azpbs.org
-        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # CA, http://www.knpb.org/
-        ('soptv.org', 'SOPTV (KSYS)'),  # CA, http://www.soptv.org
-        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # CA, http://www.klcs.org
-        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # CA, http://www.krcb.org
-        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # CA, http://kvcr.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # CO, http://www.rmpbs.org
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # CO, http://www.kenw.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # CO, http://www.kued.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # CO, http://www.newmexicopbs.org/
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # CO, http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # CO, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # CO, http://www.kbyutv.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CT, http://cptv.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # CT, http://www.thirteen.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # CT, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # CT, http://www.wgby.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # CT, http://www.njtvonline.org/
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # CT, http://www.ripbs.org/home/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # CT, http://www.wliw.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DE, http://www.mpt.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # DE, http://www.weta.org
-        ('whyy.org', 'WHYY (WHYY)'),  # DE, http://www.whyy.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # DE, http://www.njtvonline.org/
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # DE, http://www.wlvt.org/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # DE, http://www.wliw.org/
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # DC, http://www.weta.org
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DC, http://www.mpt.org
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # DC, http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # DC, http://www.whut.org
-        ('wedu.org', 'WEDU PBS (WEDU)'),  # FL, http://www.wedu.org
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # FL, http://wfsu.org/
-        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # FL, http://www.wgcu.org/
-        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # FL, http://www.wjct.org
-        ('wpbt2.org', 'WPBT2 (WPBT)'),  # FL, http://www.wpbt2.org
-        ('wsre.org', 'WSRE (WSRE)'),  # FL, http://www.wsre.org
-        ('wucftv.org', 'WUCF TV (WUCF)'),  # FL, http://wucftv.org
-        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # FL, http://www.wuft.org
-        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # FL, http://www.wxel.org/home/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # FL, http://aptv.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # FL, http://www.gpb.org/
-        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # FL, http://www.wlrn.org/
-        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # FL, http://wusf.org/
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # FL, http://www.njtvonline.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # GA, http://www.gpb.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # GA, http://aptv.org/
-        ('scetv.org', 'ETV (WRLK)'),  # GA, http://www.scetv.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # GA, http://www.unctv.org/
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # GA, http://wfsu.org/
-        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # GA, http://www.wjct.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # GA, http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # GA, http://pba.org/
-        ('pbsguam.org', 'PBS Guam (KGTF)'),  # GU, http://www.pbsguam.org/
-        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # HI, http://www.pbshawaii.org/
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # ID, http://idahoptv.org
-        ('ksps.org', 'KSPS (KSPS)'),  # ID, http://www.ksps.org/home/
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # ID, http://www.kued.org
-        ('opb.org', 'OPB (KOPB)'),  # ID, http://www.opb.org
-        ('soptv.org', 'SOPTV (KSYS)'),  # ID, http://www.soptv.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # ID, http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # ID, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # ID, http://www.kbyutv.org/
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # ID, http://www.kwsu.org
-        ('illinois.edu', 'WILL-TV (WILL)'),  # IL, http://will.illinois.edu/
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IL, http://www.wsec.tv
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IL, http://www.wsiu.org/
-        ('wttw.com', 'WTTW11 (WTTW)'),  # IL, http://www.wttw.com/
-        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # IL, http://www.wtvp.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IL, http://www.iptv.org/
-        ('ket.org', 'KET (WKLE)'),  # IL, http://www.ket.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # IL, http://www.ninenet.org
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IL, http://wfwa.org/
-        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IL, http://www.wfyi.org
-        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # IL, http://www.mptv.org
-        ('wnin.org', 'WNIN (WNIN)'),  # IL, http://www.wnin.org/
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IL, http://www.wnit.org/
-        ('wpt.org', 'WPT (WPNE)'),  # IL, http://www.wpt.org/
-        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IL, http://wvut.org/
-        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IL, http://www.weiu.net
-        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IL, http://www.wqpt.org
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IL, http://www.wycc.org
-        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IL, http://lakeshorepublicmedia.org/
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IN, http://wfwa.org/
-        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IN, http://www.wfyi.org
-        ('wipb.org', 'WIPB-TV (WIPB)'),  # IN, http://wipb.org
-        ('wnin.org', 'WNIN (WNIN)'),  # IN, http://www.wnin.org/
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IN, http://www.wnit.org/
-        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # IN, http://indianapublicmedia.org/tv/
-        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IN, http://wvut.org/
-        ('cetconnect.org', 'CET  (WCET)'),  # IN, http://www.cetconnect.org
-        ('ket.org', 'KET (WKLE)'),  # IN, http://www.ket.org/
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # IN, http://www.thinktv.org
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # IN, http://wbgu.org
-        ('wgvu.org', 'WGVU TV (WGVU)'),  # IN, http://www.wgvu.org/
-        ('illinois.edu', 'WILL-TV (WILL)'),  # IN, http://will.illinois.edu/
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IN, http://www.wsec.tv
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IN, http://www.wsiu.org/
-        ('wttw.com', 'WTTW11 (WTTW)'),  # IN, http://www.wttw.com/
-        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IN, http://lakeshorepublicmedia.org/
-        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IN, http://www.weiu.net
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IN, http://www.wycc.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IA, http://www.iptv.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # IA, http://netnebraska.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # IA, http://www.pioneer.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # IA, http://www.sdpb.org
-        ('tpt.org', 'TPT (KTCA)'),  # IA, http://www.tpt.org
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IA, http://www.wsec.tv
-        ('wpt.org', 'WPT (WPNE)'),  # IA, http://www.wpt.org/
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # IA, http://www.ksmq.org/
-        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IA, http://www.wqpt.org
-        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # KS, http://www.kpts.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # KS, http://ktwu.org
-        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # KS, http://www.shptv.org
-        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # KS, http://kcpt.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # KS, http://netnebraska.org
-        ('oeta.tv', 'OETA (KETA)'),  # KS, http://www.oeta.tv
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # KS, http://www.optv.org/
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # KS, http://www.rmpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # KS, http://www.cpt12.org/
-        ('ket.org', 'KET (WKLE)'),  # KY, http://www.ket.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # KY, http://www.blueridgepbs.org/
-        ('cetconnect.org', 'CET  (WCET)'),  # KY, http://www.cetconnect.org
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # KY, http://easttennesseepbs.org
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # KY, http://www.wnpt.org
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # KY, http://www.thinktv.org
-        ('wcte.org', 'WCTE-TV (WCTE)'),  # KY, http://www.wcte.org
-        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # KY, http://wljt.org/
-        ('wnin.org', 'WNIN (WNIN)'),  # KY, http://www.wnin.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # KY, http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # KY, http://woub.org/tv/index.php?section=5
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # KY, http://www.wsiu.org/
-        ('wvpublic.org', 'WVPB (WVPB)'),  # KY, http://wvpublic.org/
-        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # KY, http://www.wkyupbs.org
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # LA, http://www.lpb.org/
-        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # LA, http://www.wyes.org
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # LA, http://www.aetn.org/
-        ('kera.org', 'KERA 13 (KERA)'),  # LA, http://www.kera.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # LA, http://www.mpbonline.org
-        ('mpbn.net', 'MPBN (WCBB)'),  # ME, http://www.mpbn.net/
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # ME, http://www.mountainlake.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # ME, http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # ME, http://www.vpt.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # ME, http://wgbh.org
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # MD, http://www.mpt.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # MD, http://www.thirteen.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # MD, http://www.weta.org
-        ('whyy.org', 'WHYY (WHYY)'),  # MD, http://www.whyy.org
-        ('witf.org', 'witf (WITF)'),  # MD, http://www.witf.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # MD, http://www.wqed.org/
-        ('wvpublic.org', 'WVPB (WVPB)'),  # MD, http://wvpublic.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # MD, http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # MD, http://www.whut.org
-        ('wliw.org', 'WLIW21 (WLIW)'),  # MD, http://www.wliw.org/
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # MA, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # MA, http://www.wgby.org
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # MA, http://cptv.org
-        ('nhptv.org', 'NHPTV (WENH)'),  # MA, http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # MA, http://www.vpt.org
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # MA, http://www.wmht.org/home/
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # MA, http://www.ripbs.org/home/
-        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # MI, http://www.deltabroadcasting.org
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MI, http://www.dptv.org/
-        ('wcmu.org', 'CMU Public Television (WCMU)'),  # MI, http://www.wcmu.org
-        ('wgvu.org', 'WGVU TV (WGVU)'),  # MI, http://www.wgvu.org/
-        ('wkar.org', 'WKAR-TV (WKAR)'),  # MI, http://wkar.org/
-        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # MI, http://wnmutv.nmu.edu
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MI, http://www.wdse.org/
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # MI, http://wbgu.org
-        ('wgte.org', 'WGTE TV (WGTE)'),  # MI, http://www.wgte.org
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # MI, http://www.wnit.org/
-        ('wpt.org', 'WPT (WPNE)'),  # MI, http://www.wpt.org/
-        ('wttw.com', 'WTTW11 (WTTW)'),  # MI, http://www.wttw.com/
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # MI, http://www.wycc.org
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # MN, http://www.lakelandptv.org
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MN, http://www.wdse.org/
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # MN, http://www.pioneer.org
-        ('tpt.org', 'TPT (KTCA)'),  # MN, http://www.tpt.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MN, http://www.iptv.org/
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MN, http://www.prairiepublic.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # MN, http://www.ninenet.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # MN, http://www.sdpb.org
-        ('wpt.org', 'WPT (WPNE)'),  # MN, http://www.wpt.org/
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # MN, http://www.ksmq.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # MS, http://www.mpbonline.org
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MS, http://www.aetn.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # MS, http://aptv.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MS, http://www.wkno.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # MS, http://www.lpb.org/
-        ('wsre.org', 'WSRE (WSRE)'),  # MS, http://www.wsre.org
-        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # MS, http://www.wyes.org
-        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # MO, http://kcpt.org/
-        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # MO, http://www.kmos.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # MO, http://www.ninenet.org
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # MO, http://www.optv.org/
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MO, http://www.aetn.org/
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MO, http://www.dptv.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MO, http://www.iptv.org/
-        ('ket.org', 'KET (WKLE)'),  # MO, http://www.ket.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MO, http://www.wkno.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # MO, http://ktwu.org
-        ('netnebraska.org', 'NET1 (KUON)'),  # MO, http://netnebraska.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MO, http://www.rmpbs.org
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # MO, http://www.wsec.tv
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # MO, http://www.wsiu.org/
-        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # MT, http://montanapbs.org
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # MT, http://www.azpbs.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # MT, http://idahoptv.org
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MT, http://www.prairiepublic.org/
-        ('ksps.org', 'KSPS (KSPS)'),  # MT, http://www.ksps.org/home/
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MT, http://www.rmpbs.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # MT, http://www.sdpb.org
-        ('netnebraska.org', 'NET1 (KUON)'),  # NE, http://netnebraska.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # NE, http://www.iptv.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # NE, http://ktwu.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # NE, http://www.pioneer.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NE, http://www.rmpbs.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # NE, http://www.sdpb.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # NE, http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NE, http://www.cpt12.org/
-        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # NV, http://www.knpb.org/
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # NV, http://vegaspbs.org/
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # NV, http://www.azpbs.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # NV, http://www.kued.org
-        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # NV, http://www.pbssocal.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NV, http://www.kbyutv.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # NH, http://nhptv.org/
-        ('scetv.org', 'ETV (WRLK)'),  # NH, http://www.scetv.org
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NH, http://www.mountainlake.org/
-        ('mpbn.net', 'MPBN (WCBB)'),  # NH, http://www.mpbn.net/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # NH, http://www.vpt.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NH, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # NH, http://www.wgby.org
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NJ, http://cptv.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NJ, http://www.thirteen.org
-        ('whyy.org', 'WHYY (WHYY)'),  # NJ, http://www.whyy.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NJ, http://www.njtvonline.org/
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # NJ, http://www.wlvt.org/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # NJ, http://www.wliw.org/
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # NM, http://www.kenw.org
-        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # NM, http://www.krwg.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # NM, http://www.newmexicopbs.org/
-        ('azpm.org', 'KUAT 6 (KUAT)'),  # NM, http://www.azpm.org/
-        ('panhandlepbs.org', 'KACV (KACV)'),  # NM, http://www.panhandlepbs.org/home/
-        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # NM, www.kcostv.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # NM, http://www.kued.org
-        ('oeta.tv', 'OETA (KETA)'),  # NM, http://www.oeta.tv
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NM, http://www.rmpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NM, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NM, http://www.kbyutv.org/
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NY, http://www.mountainlake.org/
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NY, http://www.thirteen.org
-        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # NY, http://www.wcny.org
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # NY, http://www.wmht.org/home/
-        ('wned.org', 'WNED (WNED)'),  # NY, http://www.wned.org/
-        ('wpbstv.org', 'WPBS (WPBS)'),  # NY, http://www.wpbstv.org
-        ('wskg.org', 'WSKG Public TV (WSKG)'),  # NY, http://wskg.org
-        ('wxxi.org', 'WXXI (WXXI)'),  # NY, http://wxxi.org
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NY, http://cptv.org
-        ('nhptv.org', 'NHPTV (WENH)'),  # NY, http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # NY, http://www.vpt.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NY, http://wgbh.org
-        ('wpsu.org', 'WPSU (WPSU)'),  # NY, http://www.wpsu.org
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # NY, http://www.wqln.org
-        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # NY, http://www.wvia.org/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # NY, http://www.wliw.org/
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NY, http://www.njtvonline.org/
-        ('unctv.org', 'UNC-TV (WUNC)'),  # NC, http://www.unctv.org/
-        ('wtvi.org', 'WTVI (WTVI)'),  # NC, http://www.wtvi.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # NC, http://www.blueridgepbs.org/
-        ('scetv.org', 'ETV (WRLK)'),  # NC, http://www.scetv.org
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # NC, http://www.gpb.org/
-        ('whro.org', 'WHRO (WHRO)'),  # NC, http://whro.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # NC, http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # NC, http://pba.org/
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # ND, http://www.prairiepublic.org/
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # ND, http://www.sdpb.org
-        ('cetconnect.org', 'CET  (WCET)'),  # OH, http://www.cetconnect.org
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # OH, http://www.thinktv.org
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # OH, http://wbgu.org
-        ('wgte.org', 'WGTE TV (WGTE)'),  # OH, http://www.wgte.org
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # OH, http://www.WesternReservePublicMedia.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # OH, http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # OH, http://woub.org/tv/index.php?section=5
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # OH, http://www.wviz.org/
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # OH, http://www.dptv.org/
-        ('ket.org', 'KET (WKLE)'),  # OH, http://www.ket.org/
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # OH, http://wfwa.org/
-        ('wipb.org', 'WIPB-TV (WIPB)'),  # OH, http://wipb.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # OH, http://www.wqed.org/
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # OH, http://www.wqln.org
-        ('wvpublic.org', 'WVPB (WVPB)'),  # OH, http://wvpublic.org/
-        ('oeta.tv', 'OETA (KETA)'),  # OK, http://www.oeta.tv
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # OK, http://www.aetn.org/
-        ('panhandlepbs.org', 'KACV (KACV)'),  # OK, http://www.panhandlepbs.org/home/
-        ('kera.org', 'KERA 13 (KERA)'),  # OK, http://www.kera.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # OK, http://www.lpb.org/
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # OK, http://www.optv.org/
-        ('opb.org', 'OPB (KOPB)'),  # OR, http://www.opb.org
-        ('soptv.org', 'SOPTV (KSYS)'),  # OR, http://www.soptv.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # OR, http://idahoptv.org
-        ('kcts9.org', 'KCTS 9 (KCTS)'),  # OR, http://kcts9.org/
-        ('keet.org', 'KEET TV (KEET)'),  # OR, http://www.keet.org
-        ('ksps.org', 'KSPS (KSPS)'),  # OR, http://www.ksps.org/home/
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # OR, http://www.kwsu.org
-        ('whyy.org', 'WHYY (WHYY)'),  # PA, http://www.whyy.org
-        ('witf.org', 'witf (WITF)'),  # PA, http://www.witf.org
-        ('wpsu.org', 'WPSU (WPSU)'),  # PA, http://www.wpsu.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # PA, http://www.wqed.org/
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # PA, http://www.wqln.org
-        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # PA, http://www.wvia.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # PA, http://cptv.org
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # PA, http://www.mpt.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # PA, http://www.thirteen.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # PA, http://www.weta.org
-        ('wned.org', 'WNED (WNED)'),  # PA, http://www.wned.org/
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # PA, http://www.WesternReservePublicMedia.org/
-        ('wskg.org', 'WSKG Public TV (WSKG)'),  # PA, http://wskg.org
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # PA, http://www.wviz.org/
-        ('wvpublic.org', 'WVPB (WVPB)'),  # PA, http://wvpublic.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # PA, http://www.wvpt.net
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # PA, http://www.wlvt.org/
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # PA, http://www.njtvonline.org/
-        ('whut.org', 'Howard University Television (WHUT)'),  # PA, http://www.whut.org
-        ('wliw.org', 'WLIW21 (WLIW)'),  # PA, http://www.wliw.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # RI, http://cptv.org
-        ('nhptv.org', 'NHPTV (WENH)'),  # RI, http://nhptv.org/
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # RI, http://wgbh.org
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # RI, http://www.ripbs.org/home/
-        ('scetv.org', 'ETV (WRLK)'),  # SC, http://www.scetv.org
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # SC, http://www.gpb.org/
-        ('unctv.org', 'UNC-TV (WUNC)'),  # SC, http://www.unctv.org/
-        ('wtvi.org', 'WTVI (WTVI)'),  # SC, http://www.wtvi.org/
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # SC, http://pba.org/
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # SD, http://www.sdpb.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # SD, http://www.iptv.org/
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # SD, http://www.prairiepublic.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # SD, http://netnebraska.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # SD, http://www.pioneer.org
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # TN, http://easttennesseepbs.org
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # TN, http://www.wkno.org/
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TN, http://www.wnpt.org
-        ('wcte.org', 'WCTE-TV (WCTE)'),  # TN, http://www.wcte.org
-        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # TN, http://wljt.org/
-        ('wtcitv.org', 'WTCI (WTCI)'),  # TN, http://www.wtcitv.org
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # TN, http://aptv.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # TN, http://www.blueridgepbs.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # TN, http://www.gpb.org/
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # TN, http://www.lakelandptv.org
-        ('ket.org', 'KET (WKLE)'),  # TN, http://www.ket.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # TN, http://www.mpbonline.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # TN, http://www.unctv.org/
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # TN, http://www.WesternReservePublicMedia.org/
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # TN, http://www.wsiu.org/
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # TN, http://www.wviz.org/
-        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # TN, http://www.wkyupbs.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # TN, http://pba.org/
-        ('basinpbs.org', 'Basin PBS (KPBT)'),  # TX, http://www.basinpbs.org
-        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # TX, http://www.houstonpublicmedia.org/
-        ('panhandlepbs.org', 'KACV (KACV)'),  # TX, http://www.panhandlepbs.org/home/
-        ('tamu.edu', 'KAMU - TV (KAMU)'),  # TX, http://KAMU.tamu.edu
-        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # TX, www.kcostv.org
-        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # TX, http://www.kedt.org
-        ('kera.org', 'KERA 13 (KERA)'),  # TX, http://www.kera.org/
-        ('klrn.org', 'KLRN (KLRN)'),  # TX, http://www.klrn.org
-        ('klru.org', 'KLRU (KLRU)'),  # TX, http://www.klru.org
-        ('kmbh.org', 'KMBH-TV (KMBH)'),  # TX, http://www.kmbh.org
-        ('knct.org', 'KNCT (KNCT)'),  # TX, http://www.knct.org
-        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # TX, http://www.ktxt.org
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # TX, http://www.aetn.org/
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # TX, http://www.kenw.org
-        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # TX, http://www.krwg.org
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # TX, http://www.lpb.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # TX, http://netnebraska.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # TX, http://www.newmexicopbs.org/
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TX, http://www.wnpt.org
-        ('oeta.tv', 'OETA (KETA)'),  # TX, http://www.oeta.tv
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # TX, http://www.rmpbs.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # UT, http://www.kued.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # UT, http://idahoptv.org
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # UT, http://vegaspbs.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # UT, http://www.kbyutv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # VT, http://www.vpt.org
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # VT, http://www.mountainlake.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # VT, http://nhptv.org/
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # VT, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # VT, http://www.wgby.org
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # VT, http://www.wmht.org/home/
-        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # VI, http://www.wtjx.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # VA, http://www.blueridgepbs.org/
-        ('ideastations.org', 'WCVE PBS (WCVE)'),  # VA, http://ideastations.org/
-        ('whro.org', 'WHRO (WHRO)'),  # VA, http://whro.org
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # VA, http://www.wvpt.net
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # VA, http://easttennesseepbs.org
-        ('ket.org', 'KET (WKLE)'),  # VA, http://www.ket.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # VA, http://www.mpt.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # VA, http://www.unctv.org/
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # VA, http://www.weta.org
-        ('whut.org', 'Howard University Television (WHUT)'),  # VA, http://www.whut.org
-        ('kcts9.org', 'KCTS 9 (KCTS)'),  # WA, http://kcts9.org/
-        ('ksps.org', 'KSPS (KSPS)'),  # WA, http://www.ksps.org/home/
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WA, http://idahoptv.org
-        ('opb.org', 'OPB (KOPB)'),  # WA, http://www.opb.org
-        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # WA, http://kbtc.org
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # WA, http://www.kwsu.org
-        ('wvpublic.org', 'WVPB (WVPB)'),  # WV, http://wvpublic.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # WV, http://www.blueridgepbs.org/
-        ('ket.org', 'KET (WKLE)'),  # WV, http://www.ket.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # WV, http://www.mpt.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # WV, http://www.weta.org
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # WV, http://www.WesternReservePublicMedia.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # WV, http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # WV, http://woub.org/tv/index.php?section=5
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # WV, http://www.wqed.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # WV, http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # WV, http://www.whut.org
-        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # WI, http://www.mptv.org
-        ('wpt.org', 'WPT (WPNE)'),  # WI, http://www.wpt.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # WI, http://www.iptv.org/
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # WI, http://www.lakelandptv.org
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # WI, http://www.wdse.org/
-        ('tpt.org', 'TPT (KTCA)'),  # WI, http://www.tpt.org
-        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # WI, http://wnmutv.nmu.edu
-        ('wttw.com', 'WTTW11 (WTTW)'),  # WI, http://www.wttw.com/
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # WI, http://www.ksmq.org/
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # WI, http://www.wycc.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # WY, http://www.wyomingpbs.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WY, http://idahoptv.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # WY, http://www.kued.org
-        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # WY, http://montanapbs.org
-        ('netnebraska.org', 'NET1 (KUON)'),  # WY, http://netnebraska.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # WY, http://www.rmpbs.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # WY, http://www.sdpb.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # WY, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # WY, http://www.kbyutv.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
+        ('wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
+        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
+        ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
+        ('ket.org', 'KET (WKLE)'),  # http://www.ket.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
+        ('oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
+        ('keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
+        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
+        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
+        ('kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
+        ('kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
+        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
+        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
+        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
+        ('soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
+        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
+        ('whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
+        ('wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
+        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
+        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        ('wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
+        ('wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
+        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
+        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
+        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
+        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
+        ('scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
+        ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
+        ('ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
+        ('opb.org', 'OPB (KOPB)'),  # http://www.opb.org
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
+        ('illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
+        ('wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
+        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
+        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
+        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
+        ('wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
+        ('wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
+        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
+        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
+        ('wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
+        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        ('wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
+        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
+        ('cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
+        ('wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
+        ('tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
+        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
+        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
+        ('wcte.org', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
+        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
+        ('wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
+        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
+        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        ('kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
+        ('mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
+        ('witf.org', 'witf (WITF)'),  # http://www.witf.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
+        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
+        ('wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
+        ('wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
+        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
+        ('wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
+        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
+        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
+        ('panhandlepbs.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
+        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
+        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
+        ('wned.org', 'WNED (WNED)'),  # http://www.wned.org/
+        ('wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
+        ('wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
+        ('wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
+        ('wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
+        ('wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
+        ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
+        ('kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
+        ('basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
+        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
+        ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        ('klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
+        ('klru.org', 'KLRU (KLRU)'),  # http://www.klru.org
+        ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
+        ('ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
+        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
     )
 
     IE_NAME = 'pbs'

From 6c7b26e13f1aebe7bd19a7899b01e49b052febe5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Dec 2015 21:28:04 +0600
Subject: [PATCH 2079/2145] [pbs] Make URLs lowercase

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6aae7f028..d71d7eb76 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -159,7 +159,7 @@ class PBSIE(InfoExtractor):
         ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
         ('wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
         ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        ('westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
         ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
         ('kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
         ('basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org

From 374c761e77b09b07918f525ad74724650309015f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20M=20Mart=C3=ADnez?= <joksnet@gmail.com>
Date: Wed, 9 Dec 2015 20:56:00 -0300
Subject: [PATCH 2080/2145] [FFmpegPostProcessor] Default of prefer ffmpeg

When no `downloader` is passed to `FFmpegPostProcessor`
an exception was raised trying to get the prefer ffmpeg param.

    AttributeError: 'NoneType' object has no attribute 'params'

This fixes and defaults to `False`.
---
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5ed723bc6..daca5d814 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -52,7 +52,7 @@ class FFmpegPostProcessor(PostProcessor):
 
     def _determine_executables(self):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
+        prefer_ffmpeg = False
 
         self.basename = None
         self.probe_basename = None
@@ -60,6 +60,7 @@ class FFmpegPostProcessor(PostProcessor):
         self._paths = None
         self._versions = None
         if self._downloader:
+            prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
             location = self._downloader.params.get('ffmpeg_location')
             if location is not None:
                 if not os.path.exists(location):

From 46047c58d033f71839477d338e128b459d4d4376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Haberthu=CC=88r?= <email@davidhaberthuer.ch>
Date: Thu, 10 Dec 2015 13:17:31 +0100
Subject: [PATCH 2081/2145] Updating README.md

- Harmonizing mentions of **youtube-dl** in the text
- Removing unnecessary Markdown markup for headers
- Adding some links
---
 README.md | 110 +++++++++++++++++++++++++++---------------------------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/README.md b/README.md
index e39f71281..7a4a4f50f 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-youtube-dl - download videos from youtube.com or other video platforms
+**youtube-dl** - download videos from youtube.com or other video platforms
 
 - [INSTALLATION](#installation)
 - [DESCRIPTION](#description)
@@ -35,7 +35,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [**youtube-dl** Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
@@ -405,18 +405,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
+You can configure **youtube-dl** by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file **youtube-dl** will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular **youtube-dl** run.
 
-### Authentication with `.netrc` file ###
+### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every **youtube-dl** execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -430,7 +430,7 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to **youtube-dl** or place it in the [configuration file](#configuration).
 
 On Windows you may also need to setup the `%HOME%` environment variable manually.
 
@@ -464,14 +464,14 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
+By default **youtube-dl** tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 **youtube-dl** uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use **youtube-dl** to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, **youtube-dl** still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to **youtube-dl** 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run **youtube-dl**.
 
 # VIDEO SELECTION
 
@@ -501,7 +501,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed **youtube-dl** using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, **youtube-dl** releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we **youtube-dl** developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -519,25 +519,25 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 ### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
 
-YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
+YouTube changed their playlist format in March 2014 and later on, so you'll need at least **youtube-dl** 2014.07.25 to download all YouTube videos.
 
-If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed **youtube-dl** with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass `-citw`?
 
-By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
+By default, **youtube-dl** intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
 ### Can you please put the `-b` option back?
 
-Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
+Most people asking this question are not aware that **youtube-dl** now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and **youtube-dl** will try to download it.
 
 ### I get HTTP error 402 when trying to download a video. What's this?
 
-Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
+Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart **youtube-dl**.
 
 ### Do I need any other programs?
 
-youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
+**youtube-dl** works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. **youtube-dl** will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
 Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
@@ -547,25 +547,25 @@ Once the video is fully downloaded, use any video player, such as [vlc](http://w
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
-It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by **youtube-dl**.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
 Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
-If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
+If you want to play the video on a machine that is not running **youtube-dl**, you can relay the video content from the machine that runs **youtube-dl**. You can use `-o -` to let **youtube-dl** stream a video to stdout, or simply allow the player to download the files written by **youtube-dl** in turn.
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
 
-### ERROR: unable to download video ###
+### ERROR: unable to download video
 
-YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
 
-### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
+### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command`
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to **youtube-dl**. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -581,13 +581,13 @@ For Windows you have to use the double quotes:
 
 ### ExtractorError: Could not find JS function u'OF'
 
-In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
 These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--source-address` options](#network-options) to select another IP address.
 
-### SyntaxError: Non-ASCII character ###
+### SyntaxError: Non-ASCII character
 
 The error
 
@@ -598,7 +598,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 (#342) **youtube-dl** is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 
@@ -606,17 +606,17 @@ To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistri
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
 
-If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+If you put **youtube-dl** and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
 
-To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+To make a different directory work - either for ffmpeg, or for **youtube-dl**, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
 
-From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+From then on, after restarting your shell, you will be able to access both **youtube-dl** and ffmpeg (and **youtube-dl** will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
 ### How do I put downloads into a specific folder?
 
 Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
 
-### How do I download a video starting with a `-` ?
+### How do I download a video starting with a `-`?
 
 Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
 
@@ -627,41 +627,41 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
+Passing cookies to **youtube-dl** is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
-As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
+As a matter of policy (as well as legality), **youtube-dl** does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to **youtube-dl**.
 
-A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into **youtube-dl**. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
 
 Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
 
 ### How can I speed up work on my issue?
 
-(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
+(Also known as: Help, my important issue not being solved!) The **youtube-dl** core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
 
-First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel.
+First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the **youtube-dl** project has grown too large to use personal email as an effective communication channel.
 
-Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
+Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the **youtube-dl** developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
 
 If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so).
 
-Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
+Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in **youtube-dl** version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and **youtube-dl** reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
-It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+It is *not* possible to detect whether a URL is supported or not. That's because **youtube-dl** contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
-If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+If you want to find out whether a given URL is supported, simply call **youtube-dl** with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run **youtube-dl** on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build **youtube-dl** and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
 
-To run youtube-dl as a developer, you don't need to build anything either. Simply execute
+To run **youtube-dl** as a developer, you don't need to build anything either. Simply execute
 
     python -m youtube_dl
 
@@ -671,7 +671,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
-If you want to create a build of youtube-dl yourself, you'll need
+If you want to create a build of **youtube-dl** yourself, you'll need
 
 * python
 * make
@@ -744,9 +744,9 @@ In any case, thank you very much for your contributions!
 
 # EMBEDDING YOUTUBE-DL
 
-youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+**youtube-dl** makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
 
-From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
+From a Python program, you can embed **youtube-dl** in a more powerful fashion, like this:
 
 ```python
 from __future__ import unicode_literals
@@ -757,7 +757,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept **youtube-dl**'s output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
@@ -798,7 +798,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
+Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](http://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
@@ -832,11 +832,11 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of **youtube-dl** when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
-**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `http://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `http://www.youtube.com/`) is *not* an example URL.
 
 ###  Are you using the latest version?
 
@@ -844,7 +844,7 @@ Before reporting any issue, type `youtube-dl -U`. This should report that you're
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/rg3/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
 
 ###  Why are existing options not enough?
 
@@ -860,7 +860,7 @@ We are then presented with a very complicated request when the original problem
 
 Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones.
 
-In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
+In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of **youtube-dl** that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
 
 ###  Is anyone going to need the feature?
 
@@ -868,10 +868,10 @@ Only post features that you (or an incapacitated friend you can personally talk
 
 ###  Is your question about youtube-dl?
 
-It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.
+It may sound strange, but some bug reports we receive are completely unrelated to **youtube-dl** and relate to a different or even the reporter's own application. Please make sure that you are actually using **youtube-dl**. If you are using a UI for **youtube-dl**, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for **youtube-dl** fails in some way you believe is related to **youtube-dl**, by all means, go ahead and report the bug.
 
 # COPYRIGHT
 
-youtube-dl is released into the public domain by the copyright holders.
+**youtube-dl** is released into the public domain by the copyright holders.
 
-This README file was originally written by Daniel Bolton (<https://github.com/dbbolton>) and is likewise released into the public domain.
+This README file was originally written by [Daniel Bolton](https://github.com/dbbolton) and is likewise released into the public domain.

From a1b8d815f5716783f5ec9c2b4f60dcdd4d8192d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Haberthu=CC=88r?= <email@davidhaberthuer.ch>
Date: Thu, 10 Dec 2015 13:45:53 +0100
Subject: [PATCH 2082/2145] Reverting markup changes

---
 README.md | 92 +++++++++++++++++++++++++++----------------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/README.md b/README.md
index 7a4a4f50f..5c3915d64 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-**youtube-dl** - download videos from youtube.com or other video platforms
+youtube-dl - download videos from youtube.com or other video platforms
 
 - [INSTALLATION](#installation)
 - [DESCRIPTION](#description)
@@ -35,7 +35,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [**youtube-dl** Download Page](https://rg3.github.io/youtube-dl/download.html).
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
@@ -405,18 +405,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure **youtube-dl** by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file **youtube-dl** will always extract the audio, not copy the mtime and use a proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable the configuration file for a particular **youtube-dl** run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every **youtube-dl** execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -430,7 +430,7 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with the `.netrc` file you should pass `--netrc` to **youtube-dl** or place it in the [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
 On Windows you may also need to setup the `%HOME%` environment variable manually.
 
@@ -464,14 +464,14 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default **youtube-dl** tries to download the best quality, but sometimes you may want to download in a different format.
+By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 **youtube-dl** uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use **youtube-dl** to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, **youtube-dl** still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to **youtube-dl** 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run **youtube-dl**.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 
@@ -501,7 +501,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed **youtube-dl** using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, **youtube-dl** releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we **youtube-dl** developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -519,25 +519,25 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 ### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
 
-YouTube changed their playlist format in March 2014 and later on, so you'll need at least **youtube-dl** 2014.07.25 to download all YouTube videos.
+YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
 
-If you have installed **youtube-dl** with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass `-citw`?
 
-By default, **youtube-dl** intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
+By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
 ### Can you please put the `-b` option back?
 
-Most people asking this question are not aware that **youtube-dl** now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and **youtube-dl** will try to download it.
+Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
 
 ### I get HTTP error 402 when trying to download a video. What's this?
 
-Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart **youtube-dl**.
+Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
 
 ### Do I need any other programs?
 
-**youtube-dl** works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. **youtube-dl** will detect whether avconv/ffmpeg is present and automatically pick the best option.
+youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
 Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
@@ -547,25 +547,25 @@ Once the video is fully downloaded, use any video player, such as [vlc](http://w
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
-It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by **youtube-dl**.
+It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
 Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
-If you want to play the video on a machine that is not running **youtube-dl**, you can relay the video content from the machine that runs **youtube-dl**. You can use `-o -` to let **youtube-dl** stream a video to stdout, or simply allow the player to download the files written by **youtube-dl** in turn.
+If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-YouTube has switched to a new video info format in July 2011 which is not supported by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### ERROR: unable to download video
 
-YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command`
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to **youtube-dl**. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -581,7 +581,7 @@ For Windows you have to use the double quotes:
 
 ### ExtractorError: Could not find JS function u'OF'
 
-In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
@@ -598,7 +598,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) **youtube-dl** is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 
@@ -606,11 +606,11 @@ To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistri
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
 
-If you put **youtube-dl** and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
 
-To make a different directory work - either for ffmpeg, or for **youtube-dl**, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
 
-From then on, after restarting your shell, you will be able to access both **youtube-dl** and ffmpeg (and **youtube-dl** will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
 ### How do I put downloads into a specific folder?
 
@@ -627,41 +627,41 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to **youtube-dl** is a good way to workaround login when a particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
-As a matter of policy (as well as legality), **youtube-dl** does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to **youtube-dl**.
+As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
 
-A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into **youtube-dl**. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
 
 Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
 
 ### How can I speed up work on my issue?
 
-(Also known as: Help, my important issue not being solved!) The **youtube-dl** core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
+(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
 
-First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the **youtube-dl** project has grown too large to use personal email as an effective communication channel.
+First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel.
 
-Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the **youtube-dl** developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
+Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
 
 If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so).
 
-Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in **youtube-dl** version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
+Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and **youtube-dl** reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
-It is *not* possible to detect whether a URL is supported or not. That's because **youtube-dl** contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
-If you want to find out whether a given URL is supported, simply call **youtube-dl** with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run **youtube-dl** on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build **youtube-dl** and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
 
-To run **youtube-dl** as a developer, you don't need to build anything either. Simply execute
+To run youtube-dl as a developer, you don't need to build anything either. Simply execute
 
     python -m youtube_dl
 
@@ -671,7 +671,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
-If you want to create a build of **youtube-dl** yourself, you'll need
+If you want to create a build of youtube-dl yourself, you'll need
 
 * python
 * make
@@ -744,9 +744,9 @@ In any case, thank you very much for your contributions!
 
 # EMBEDDING YOUTUBE-DL
 
-**youtube-dl** makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
 
-From a Python program, you can embed **youtube-dl** in a more powerful fashion, like this:
+From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
 ```python
 from __future__ import unicode_literals
@@ -757,7 +757,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept **youtube-dl**'s output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
@@ -832,7 +832,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of **youtube-dl** when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
@@ -860,7 +860,7 @@ We are then presented with a very complicated request when the original problem
 
 Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones.
 
-In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of **youtube-dl** that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
+In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
 
 ###  Is anyone going to need the feature?
 
@@ -868,10 +868,10 @@ Only post features that you (or an incapacitated friend you can personally talk
 
 ###  Is your question about youtube-dl?
 
-It may sound strange, but some bug reports we receive are completely unrelated to **youtube-dl** and relate to a different or even the reporter's own application. Please make sure that you are actually using **youtube-dl**. If you are using a UI for **youtube-dl**, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for **youtube-dl** fails in some way you believe is related to **youtube-dl**, by all means, go ahead and report the bug.
+It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.
 
 # COPYRIGHT
 
-**youtube-dl** is released into the public domain by the copyright holders.
+youtube-dl is released into the public domain by the copyright holders.
 
 This README file was originally written by [Daniel Bolton](https://github.com/dbbolton) and is likewise released into the public domain.

From 51094b1b0885462c19cc2c75f545e621f8a3aa0a Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Thu, 10 Dec 2015 21:42:12 +0800
Subject: [PATCH 2083/2145] add cookie and referer in headers, change the video
 url

---
 youtube_dl/extractor/youku.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 69ecc837a..d077e03db 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -6,6 +6,9 @@ import base64
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_cookiejar,
+    compat_cookies,
+    compat_urllib_request,
     compat_ord,
 )
 from ..utils import (
@@ -188,19 +191,28 @@ class YoukuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         def retrieve_data(req_url, note):
-            req = sanitized_Request(req_url)
+            
+
+            headers = {
+                    'Referer': req_url,
+                }
+            self._set_cookie('youku.com','xreferrer','http://www.youku.com')
+            req = sanitized_Request(req_url,headers=headers)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
-            return raw_data['data'][0]
+
+            return raw_data['data']['security']
+
 
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
-        basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        #basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
         if video_password:
             basic_data_url += '?password=%s' % video_password
 
@@ -208,7 +220,8 @@ class YoukuIE(InfoExtractor):
             basic_data_url,
             'Downloading JSON metadata 1')
         data2 = retrieve_data(
-            'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            #'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id,
             'Downloading JSON metadata 2')
 
         error_code = data1.get('error_code')

From 526a20bd16ed734fdff0bf56ec79537d7e4d7b96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:04:26 +0600
Subject: [PATCH 2084/2145] [pbs] Clarify member stations' URLs

---
 youtube_dl/extractor/pbs.py | 321 ++++++++++++++++++------------------
 1 file changed, 161 insertions(+), 160 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index d71d7eb76..d030236d4 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,164 +16,165 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
-        ('wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
-        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
-        ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
-        ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
-        ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
-        ('ket.org', 'KET (WKLE)'),  # http://www.ket.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
-        ('oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
-        ('keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
-        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
-        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
-        ('kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
-        ('kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
-        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
-        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
-        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
-        ('soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
-        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
-        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
-        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
-        ('whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
-        ('wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
-        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
-        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
-        ('wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
-        ('wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
-        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
-        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
-        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
-        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
-        ('scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
-        ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
-        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
-        ('ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
-        ('opb.org', 'OPB (KOPB)'),  # http://www.opb.org
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
-        ('illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
-        ('wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
-        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
-        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
-        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
-        ('wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
-        ('wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
-        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
-        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
-        ('wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
-        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
-        ('wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
-        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
-        ('cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
-        ('wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
-        ('tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
-        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
-        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
-        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
-        ('wcte.org', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
-        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
-        ('wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
-        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
-        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
-        ('kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
-        ('mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
-        ('witf.org', 'witf (WITF)'),  # http://www.witf.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
-        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
-        ('wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
-        ('wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
-        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
-        ('wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
-        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
-        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
-        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
-        ('panhandlepbs.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
-        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
-        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
-        ('wned.org', 'WNED (WNED)'),  # http://www.wned.org/
-        ('wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
-        ('wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
-        ('wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
-        ('wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
-        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
-        ('wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
-        ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
-        ('westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
-        ('kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
-        ('basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
-        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
-        ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
-        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
-        ('klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
-        ('klru.org', 'KLRU (KLRU)'),  # http://www.klru.org
-        ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
-        ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
-        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
-        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
-        ('ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
-        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
+        ('video.pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        ('video.aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
+        ('video.gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
+        ('video.mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
+        ('video.wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
+        ('video.wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
+        ('video.wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
+        ('video.wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
+        ('video.pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
+        ('video.alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
+        #('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        #('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        #('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        ('video.azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
+        ('portal.knme.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
+        ('video.vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
+        ('watch.aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
+        ('video.ket.org', 'KET (WKLE)'),  # http://www.ket.org/
+        ('video.wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
+        ('video.lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
+        ('videos.oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
+        ('video.optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
+        ('watch.wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
+        ('video.keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
+        ('pbs.kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
+        ('video.kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
+        ('video.kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
+        ('vids.kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
+        ('video.pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
+        ('video.valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
+        ('video.cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
+        ('watch.knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
+        ('video.soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
+        #('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        #('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        #('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        ('video.rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
+        ('video.kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
+        ('video.kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
+        ('video.wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
+        ('video.cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
+        ('video.kbyueleven.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
+        ('video.thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
+        ('video.wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
+        ('video.wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
+        ('watch.njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
+        #('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        ('watch.wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
+        ('video.mpt.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
+        ('watch.weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
+        ('video.whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
+        ('video.wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
+        ('video.wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
+        ('video.whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
+        ('video.wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
+        ('video.wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
+        #('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        ('video.wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
+        ('video.wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
+        ('video.wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
+        ('watch.wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
+        ('video.wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
+        ('video.wusf.usf.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
+        ('video.scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
+        ('video.unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
+        #('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        ('video.pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
+        ('video.idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
+        ('video.ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
+        ('watch.opb.org', 'OPB (KOPB)'),  # http://www.opb.org
+        ('watch.nwptv.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
+        ('video.will.illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
+        ('video.networkknowledge.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
+        ('video.wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
+        #('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        ('video.iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
+        ('video.ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
+        ('video.wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
+        ('video.wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
+        ('video.mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
+        ('video.wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
+        ('video.wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
+        ('video.wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
+        ('video.wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
+        ('video.weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
+        ('video.wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
+        ('video.wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
+        #('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        ('video.wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
+        ('video.indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
+        ('watch.cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
+        ('video.thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
+        ('video.wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
+        ('video.wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
+        ('video.netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
+        ('video.pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
+        ('watch.sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
+        ('video.tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
+        ('watch.ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
+        ('watch.kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
+        ('watch.ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
+        #('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        #('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        #('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        ('watch.easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
+        ('video.wcte.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
+        ('video.wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
+        ('video.wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
+        ('video.woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
+        ('video.wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
+        ('video.wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
+        #('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        ('video.kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
+        ('video.mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
+        ('video.mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
+        ('video.nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
+        ('video.vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
+        ('video.witf.org', 'witf (WITF)'),  # http://www.witf.org
+        ('watch.wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
+        ('video.wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
+        ('video.deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
+        ('video.dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
+        ('video.wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
+        ('video.wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
+        ('wnmuvideo.nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
+        ('video.wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
+        ('video.wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
+        ('video.lptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
+        #('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        ('video.kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
+        ('watch.montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
+        ('video.krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
+        ('video.kacvtv.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
+        ('video.kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
+        ('video.wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
+        ('video.wned.org', 'WNED (WNED)'),  # http://www.wned.org/
+        ('watch.wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
+        ('video.wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
+        ('video.wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
+        ('video.wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
+        #('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        ('on-demand.wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
+        ('video.wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
+        #('whro.org', 'WHRO (WHRO)'),  # http://whro.org
+        ('video.westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        ('video.ideastream.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
+        ('video.kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
+        ('video.basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
+        ('video.houstonpbs.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
+        #('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        #('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        ('video.klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
+        ('video.klru.tv', 'KLRU (KLRU)'),  # http://www.klru.org
+        #('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        #('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        #('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        ('video.wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
+        ('video.ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
+        ('video.kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
     )
 
     IE_NAME = 'pbs'
@@ -182,13 +183,13 @@ class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           (?:videos?|watch)\.(?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
-    ''' % '|'.join(list(zip(*_STATIONS))[0])
+    ''' % '|'.join(re.escape(p) for p in list(zip(*_STATIONS))[0])
 
     _TESTS = [
         {

From 0b534d2adc56ead4fcb085d886e165dfeaf2852f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:27:47 +0600
Subject: [PATCH 2085/2145] [dailymotion] Restrict player v5 regex (Closes
 #7826)

---
 youtube_dl/extractor/dailymotion.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index ab7f3aec4..fd6d14d23 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -122,7 +122,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(
-            [r'buildPlayer\(({.+?})\);', r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);'],
+            [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
+             r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+             r'buildPlayer\(({.+?})\);'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From cbd2ffd031ea1a3388dce96db1d8093daa3c3b77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:29:07 +0600
Subject: [PATCH 2086/2145] [dailymotion] Fix subtitles extraction

---
 youtube_dl/extractor/dailymotion.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index fd6d14d23..e260a0c59 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -174,11 +174,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             uploader_id = metadata.get('owner', {}).get('id')
 
             subtitles = {}
-            for subtitle_lang, subtitle in metadata.get('subtitles', {}).get('data', {}).items():
-                subtitles[subtitle_lang] = [{
-                    'ext': determine_ext(subtitle_url),
-                    'url': subtitle_url,
-                } for subtitle_url in subtitle.get('urls', [])]
+            subtitles_data = metadata.get('subtitles', {}).get('data', {})
+            if subtitles_data and isinstance(subtitles_data, dict):
+                for subtitle_lang, subtitle in subtitles_data.items():
+                    subtitles[subtitle_lang] = [{
+                        'ext': determine_ext(subtitle_url),
+                        'url': subtitle_url,
+                    } for subtitle_url in subtitle.get('urls', [])]
 
             return {
                 'id': video_id,

From 176ccefcd89af5393b7cab14237097c54dd0ee14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:33:40 +0600
Subject: [PATCH 2087/2145] [pbs] PEP 8

---
 youtube_dl/extractor/pbs.py | 46 ++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index d030236d4..604a0dd22 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -26,9 +26,9 @@ class PBSIE(InfoExtractor):
         ('video.wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
         ('video.pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
         ('video.alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
-        #('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
-        #('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
-        #('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        # ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        # ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        # ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
         ('video.azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
         ('portal.knme.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
         ('video.vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
@@ -49,9 +49,9 @@ class PBSIE(InfoExtractor):
         ('video.cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
         ('watch.knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
         ('video.soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
-        #('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
-        #('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
-        #('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        # ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        # ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        # ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
         ('video.rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
         ('video.kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
         ('video.kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
@@ -62,7 +62,7 @@ class PBSIE(InfoExtractor):
         ('video.wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
         ('video.wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
         ('watch.njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
-        #('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        # ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
         ('watch.wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
         ('video.mpt.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
         ('watch.weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
@@ -72,7 +72,7 @@ class PBSIE(InfoExtractor):
         ('video.whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
         ('video.wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
         ('video.wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
-        #('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        # ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
         ('video.wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
         ('video.wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
         ('video.wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
@@ -81,7 +81,7 @@ class PBSIE(InfoExtractor):
         ('video.wusf.usf.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
         ('video.scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
         ('video.unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
-        #('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        # ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
         ('video.pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
         ('video.idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
         ('video.ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
@@ -90,7 +90,7 @@ class PBSIE(InfoExtractor):
         ('video.will.illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
         ('video.networkknowledge.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
         ('video.wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
-        #('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        # ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
         ('video.iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
         ('video.ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
         ('video.wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
@@ -103,7 +103,7 @@ class PBSIE(InfoExtractor):
         ('video.weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
         ('video.wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
         ('video.wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
-        #('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        # ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
         ('video.wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
         ('video.indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
         ('watch.cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
@@ -117,9 +117,9 @@ class PBSIE(InfoExtractor):
         ('watch.ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
         ('watch.kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
         ('watch.ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
-        #('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
-        #('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
-        #('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        # ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        # ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        # ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
         ('watch.easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
         ('video.wcte.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
         ('video.wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
@@ -127,7 +127,7 @@ class PBSIE(InfoExtractor):
         ('video.woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
         ('video.wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
         ('video.wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
-        #('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        # ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
         ('video.kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
         ('video.mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
         ('video.mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
@@ -144,7 +144,7 @@ class PBSIE(InfoExtractor):
         ('video.wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
         ('video.wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
         ('video.lptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
-        #('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        # ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
         ('video.kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
         ('watch.montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
         ('video.krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
@@ -156,22 +156,22 @@ class PBSIE(InfoExtractor):
         ('video.wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
         ('video.wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
         ('video.wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
-        #('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        # ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
         ('on-demand.wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
         ('video.wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
-        #('whro.org', 'WHRO (WHRO)'),  # http://whro.org
+        # ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
         ('video.westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
         ('video.ideastream.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
         ('video.kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
         ('video.basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
         ('video.houstonpbs.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
-        #('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
-        #('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        # ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        # ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
         ('video.klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
         ('video.klru.tv', 'KLRU (KLRU)'),  # http://www.klru.org
-        #('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
-        #('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
-        #('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        # ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        # ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        # ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
         ('video.wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
         ('video.ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
         ('video.kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org

From a8abf124c80cac3f2a75a2261523a3836376888d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:54:48 +0600
Subject: [PATCH 2088/2145] [dailymotion] Add subtitles test URL for reference

---
 youtube_dl/extractor/dailymotion.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index e260a0c59..428556213 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -99,6 +99,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         {
             'url': 'http://www.dailymotion.com/video/xhza0o',
             'only_matching': True,
+        },
+        # with subtitles
+        {
+            'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
+            'only_matching': True,
         }
     ]
 

From f7b8dd63f0b99a6953f1ab0645c923ca2f1e157d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 10 Dec 2015 17:05:13 +0100
Subject: [PATCH 2089/2145] release 2015.12.10

---
 CONTRIBUTING.md        | 4 ++--
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 28aae94f8..f3fe0d432 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -34,7 +34,7 @@ For bug reports, this means that your report should contain the *complete* outpu
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
-**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `http://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `http://www.youtube.com/`) is *not* an example URL.
 
 ###  Are you using the latest version?
 
@@ -42,7 +42,7 @@ Before reporting any issue, type `youtube-dl -U`. This should report that you're
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/rg3/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
 
 ###  Why are existing options not enough?
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4f19cb6f5..3bb3abb25 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -400,7 +400,7 @@
  - **orf:tvthek**: ORF TVthek
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
- - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), KUAC (KUAC), 360 North (KTOO), KUAT 6 (KUAT), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Mississippi Public Broadcasting (WMPN), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), APT - Alabama Public Television (WBIQ), CONNECTICUT PUBLIC TELEVISION (WEDH), Arizona PBS (KAET), KNPB Channel 5 (KNPB), SOPTV (KSYS), KLCS/Channel 58 (KLCS), KRCB Television & Radio (KRCB), KVCR TV/DT/FM :: Vision for the Future (KVCR), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), KNME-TV/Channel 5 (KNME), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), Rhode Island PBS (WSBE), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), WETA Television and Radio (WETA), mpt/Maryland Public Television (WMPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WFSU-TV (WFSU), WGCU Public Media (WGCU), WJCT Public Broadcasting (WJCT), WPBT2 (WPBT), WSRE (WSRE), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), NJTV Public Media NJ (WNJT), GPB/Georgia Public Broadcasting (WGTV), APT - Alabama Public Television (WBIQ), ETV (WRLK), UNC-TV (WUNC), WFSU-TV (WFSU), WJCT Public Broadcasting (WJCT), WTCI (WTCI), WPBA/Channel 30 (WPBA), PBS Guam (KGTF), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), KUED Channel 7 (KUED), OPB (KOPB), SOPTV (KSYS), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), WTVP & WTVP.org, Public Media for Central Illinois (WTVP), Iowa Public Television/IPTV (KDIN), KET (WKLE), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), Lakeshore Public Television (WYIN), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), WIPB-TV (WIPB), WNIN (WNIN), WNIT Public Television (WNIT), WTIU (WTIU), WVUT/Channel 22 (WVUT), CET  (WCET), KET (WKLE), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), Lakeshore Public Television (WYIN), WEIU/Channel 51 (WEIU), WYCC PBS Chicago (WYCC), Iowa Public Television/IPTV (KDIN), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), Network Knowledge - WSEC/Springfield (WSEC), WPT (WPNE), KSMQ (KSMQ), WQPT-TV (WQPT), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), Smoky Hills Public Television (KOOD), KCPT Kansas City Public Television (KCPT), NET1 (KUON), OETA (KETA), Ozarks Public Television (KOZK), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KET (WKLE), Blue Ridge PBS (WBRA), CET  (WCET), East Tennessee PBS (WSJK), Nashville Public Television (WNPT), ThinkTVNetwork (WPTD), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WNIN (WNIN), WOSU TV (WOSU), WOUB/WOUC (WOUB), WSIU Public Broadcasting (WSIU), WVPB (WVPB), WKYU-PBS (WKYU), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WYES-TV/New Orleans (WYES), AETN/ARKANSAS ETV NETWORK (KETS), KERA 13 (KERA), Mississippi Public Broadcasting (WMPN), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WHYY (WHYY), witf (WITF), WQED Multimedia (WQED), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WLIW21 (WLIW), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WMHT Educational Telecommunications (WMHT), Rhode Island PBS (WSBE), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WGVU TV (WGVU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WBGU-TV (WBGU), WGTE TV (WGTE), WNIT Public Television (WNIT), WPT (WPNE), WTTW11 (WTTW), WYCC PBS Chicago (WYCC), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), Pioneer Public Television (KWCM), TPT (KTCA), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), Nine Network (KETC), SDPB Television (KUSD), WPT (WPNE), KSMQ (KSMQ), Mississippi Public Broadcasting (WMPN), AETN/ARKANSAS ETV NETWORK (KETS), APT - Alabama Public Television (WBIQ), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WSRE (WSRE), WYES-TV/New Orleans (WYES), KCPT Kansas City Public Television (KCPT), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), Nine Network (KETC), Ozarks Public Television (KOZK), AETN/ARKANSAS ETV NETWORK (KETS), WTVS Detroit Public TV (WTVS), Iowa Public Television/IPTV (KDIN), KET (WKLE), WKNO/Channel 10 (WKNO), KTWU/Channel 11 (KTWU), NET1 (KUON), Rocky Mountain PBS (KRMA), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), MontanaPBS (KUSM), Arizona PBS (KAET), Idaho Public Television (KAID), PRAIRIE PUBLIC (KFME), KSPS (KSPS), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), NET1 (KUON), Iowa Public Television/IPTV (KDIN), KTWU/Channel 11 (KTWU), Pioneer Public Television (KWCM), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KNPB Channel 5 (KNPB), Vegas PBS (KLVX), Arizona PBS (KAET), KUED Channel 7 (KUED), PBS SoCal/KOCE (KOCE), KBYU-TV (KBYU), NHPTV (WENH), ETV (WRLK), Mountain Lake PBS (WCFE), MPBN (WCBB), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), KNME-TV/Channel 5 (KNME), KUAT 6 (KUAT), KACV (KACV), KCOS/Channel 13 (KCOS), KUED Channel 7 (KUED), OETA (KETA), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Mountain Lake PBS (WCFE), Thirteen/WNET New York (WNET), WCNY/Channel 24 (WCNY), WMHT Educational Telecommunications (WMHT), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WPSU (WPSU), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), WLIW21 (WLIW), NJTV Public Media NJ (WNJT), UNC-TV (WUNC), WTVI (WTVI), Blue Ridge PBS (WBRA), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), WHRO (WHRO), WTCI (WTCI), WPBA/Channel 30 (WPBA), PRAIRIE PUBLIC (KFME), SDPB Television (KUSD), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGTE TV (WGTE), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVIZ/PBS ideastream (WVIZ), WTVS Detroit Public TV (WTVS), KET (WKLE), PBS39 Fort Wayne (WFWA), WIPB-TV (WIPB), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVPB (WVPB), OETA (KETA), AETN/ARKANSAS ETV NETWORK (KETS), KACV (KACV), KERA 13 (KERA), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Ozarks Public Television (KOZK), OPB (KOPB), SOPTV (KSYS), Idaho Public Television (KAID), KCTS 9 (KCTS), KEET TV (KEET), KSPS (KSPS), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WHYY (WHYY), witf (WITF), WPSU (WPSU), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), CONNECTICUT PUBLIC TELEVISION (WEDH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WNED (WNED), Western Reserve PBS (WNEO), WSKG Public TV (WSKG), WVIZ/PBS ideastream (WVIZ), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), PBS 39 (WLVT), NJTV Public Media NJ (WNJT), Howard University Television (WHUT), WLIW21 (WLIW), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), WGBH/Channel 2 (WGBH), Rhode Island PBS (WSBE), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), UNC-TV (WUNC), WTVI (WTVI), WPBA/Channel 30 (WPBA), SDPB Television (KUSD), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), NET1 (KUON), Pioneer Public Television (KWCM), East Tennessee PBS (WSJK), WKNO/Channel 10 (WKNO), Nashville Public Television (WNPT), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WTCI (WTCI), APT - Alabama Public Television (WBIQ), Blue Ridge PBS (WBRA), GPB/Georgia Public Broadcasting (WGTV), Lakeland Public Television (KAWE), KET (WKLE), Mississippi Public Broadcasting (WMPN), UNC-TV (WUNC), Western Reserve PBS (WNEO), WSIU Public Broadcasting (WSIU), WVIZ/PBS ideastream (WVIZ), WKYU-PBS (WKYU), WPBA/Channel 30 (WPBA), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KACV (KACV), KAMU - TV (KAMU), KCOS/Channel 13 (KCOS), KEDT/Channel 16 (KEDT), KERA 13 (KERA), KLRN (KLRN), KLRU (KLRU), KMBH-TV (KMBH), KNCT (KNCT), KTTZ-TV (KTXT), AETN/ARKANSAS ETV NETWORK (KETS), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), NET1 (KUON), KNME-TV/Channel 5 (KNME), Nashville Public Television (WNPT), OETA (KETA), Rocky Mountain PBS (KRMA), KUED Channel 7 (KUED), Idaho Public Television (KAID), Vegas PBS (KLVX), KBYU-TV (KBYU), Vermont PBS (WETK), Mountain Lake PBS (WCFE), NHPTV (WENH), WGBH/Channel 2 (WGBH), WGBY (WGBY), WMHT Educational Telecommunications (WMHT), WTJX Channel 12 (WTJX), Blue Ridge PBS (WBRA), WCVE PBS (WCVE), WHRO (WHRO), WVPT - Your Source for PBS and More! (WVPT), East Tennessee PBS (WSJK), KET (WKLE), mpt/Maryland Public Television (WMPB), UNC-TV (WUNC), WETA Television and Radio (WETA), Howard University Television (WHUT), KCTS 9 (KCTS), KSPS (KSPS), Idaho Public Television (KAID), OPB (KOPB), KBTC Public Television (KBTC), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WVPB (WVPB), Blue Ridge PBS (WBRA), KET (WKLE), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WQED Multimedia (WQED), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), Milwaukee Public Television (WMVS), WPT (WPNE), Iowa Public Television/IPTV (KDIN), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), TPT (KTCA), WNMU-TV Public TV 13 (WNMU), WTTW11 (WTTW), KSMQ (KSMQ), WYCC PBS Chicago (WYCC), Wyoming PBS (KCWC), Idaho Public Television (KAID), KUED Channel 7 (KUED), MontanaPBS (KUSM), NET1 (KUON), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU)
+ - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0f5571006..6cd748b48 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.09'
+__version__ = '2015.12.10'

From 17b786ae73848a8f26ecc6e95947c3f65edc667f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 22:59:50 +0600
Subject: [PATCH 2090/2145] [downloader/f4m] Fix malformed manifests (Closes
 #7823)

---
 youtube_dl/downloader/f4m.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 6170cc155..7819a91dc 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -15,6 +15,7 @@ from ..compat import (
 )
 from ..utils import (
     encodeFilename,
+    fix_xml_ampersands,
     sanitize_open,
     struct_pack,
     struct_unpack,
@@ -288,7 +289,10 @@ class F4mFD(FragmentFD):
         self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
         urlh = self.ydl.urlopen(man_url)
         man_url = urlh.geturl()
-        manifest = urlh.read()
+        # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+        # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
+        # and https://github.com/rg3/youtube-dl/issues/7823)
+        manifest = fix_xml_ampersands(urlh.read()).strip()
 
         doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)

From 75ed53320e4a11b648dd9ae0bea292f3d1000f00 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Dec 2015 19:08:16 +0100
Subject: [PATCH 2091/2145] [ooyala] improve extraction

---
 youtube_dl/extractor/ooyala.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 35067e271..8603fd692 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -27,6 +27,7 @@ class OoyalaBaseIE(InfoExtractor):
             'duration': float_or_none(metadata.get('duration'), 1000),
         }
 
+        urls = []
         formats = []
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
@@ -38,20 +39,28 @@ class OoyalaBaseIE(InfoExtractor):
             if cur_auth_data['authorized']:
                 for stream in cur_auth_data['streams']:
                     url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                    if url in urls:
+                        continue
+                    urls.append(url)
                     delivery_type = stream['delivery_type']
-                    if delivery_type == 'remote_asset':
-                        video_info['url'] = url
-                        return video_info
-                    if delivery_type == 'hls':
-                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                    elif delivery_type == 'hds':
-                        formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
+                    if delivery_type == 'hls' or '.m3u8' in url:
+                        m3u8_formats = self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
+                    elif delivery_type == 'hds' or '.f4m' in url:
+                        f4m_formats = self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
+                    elif '.smil' in url:
+                        smil_formats = self._extract_smil_formats(url, embed_code, fatal=False)
+                        if smil_formats:
+                            formats.extend(smil_formats)
                     else:
                         formats.append({
                             'url': url,
                             'ext': stream.get('delivery_type'),
                             'vcodec': stream.get('video_codec'),
-                            'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
+                            'format_id': delivery_type,
                             'width': int_or_none(stream.get('width')),
                             'height': int_or_none(stream.get('height')),
                             'abr': int_or_none(stream.get('audio_bitrate')),

From 4bf99ade15c2b9dc0ec0ce4e6127d4cd56f87896 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 14:25:01 +0800
Subject: [PATCH 2092/2145] [vevo] Catch the georestriction message (#7802)

---
 youtube_dl/extractor/vevo.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 571289421..61813301a 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -81,6 +81,9 @@ class VevoIE(InfoExtractor):
         if webpage is False:
             self._oauth_token = None
         else:
+            if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
+                raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+
             self._oauth_token = self._search_regex(
                 r'access_token":\s*"([^"]+)"',
                 webpage, 'access token', fatal=False)

From 4b942883019968c556a6c6b989b346cecc474ac2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 17:32:29 +0800
Subject: [PATCH 2093/2145] [vevo] Use _match_id

---
 youtube_dl/extractor/vevo.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 61813301a..1d834970c 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -169,8 +169,7 @@ class VevoIE(InfoExtractor):
             preference=0)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(json_url, video_id)

From fdf01663d1fd362d9886dbe2e4136bca23cc1543 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Fri, 11 Dec 2015 17:48:40 +0800
Subject: [PATCH 2094/2145] able to download first part of the video, but fail
 in the left part

---
 youtube_dl/extractor/youku.py | 57 +++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d077e03db..155286fc5 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import base64
 
 from .common import InfoExtractor
@@ -84,17 +85,18 @@ class YoukuIE(InfoExtractor):
             return bytes(s)
 
         sid, token = yk_t(
-            b'becaf9be', base64.b64decode(data2['ep'].encode('ascii'))
+            b'becaf9be', base64.b64decode(data2['security']['encrypt_string'].encode('ascii'))
         ).decode('ascii').split('_')
 
         # get oip
-        oip = data2['ip']
+        oip = data2['security']['ip']
 
         # get fileid
         string_ls = list(
             'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
         shuffled_string_ls = []
-        seed = data1['seed']
+        seed = data1.get('seed')
+        seed = 0
         N = len(string_ls)
         for ii in range(N):
             seed = (seed * 0xd3 + 0x754f) % 0x10000
@@ -103,15 +105,19 @@ class YoukuIE(InfoExtractor):
             del string_ls[idx]
 
         fileid_dict = {}
-        for format in data1['streamtypes']:
-            streamfileid = [
-                int(i) for i in data1['streamfileids'][format].strip('*').split('*')]
-            fileid = ''.join(
-                [shuffled_string_ls[i] for i in streamfileid])
-            fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
+        for stream in data1['stream']:
+            format = stream.get('stream_type')
+            #streamfileid = [
+            #    int(i) for i in data1['stream']['streamfileids']#[format].strip('*').split('*')]
+            #fileid = ''.join(
+            #    [shuffled_string_ls[i] for i in streamfileid])
+            fileid = stream['stream_fileid']
+            fileid_dict[format] = fileid
+            #fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
 
         def get_fileid(format, n):
-            fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            #fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            fileid = fileid_dict[format]
             return fileid
 
         # get ep
@@ -126,15 +132,16 @@ class YoukuIE(InfoExtractor):
 
         # generate video_urls
         video_urls_dict = {}
-        for format in data1['streamtypes']:
+        for stream in data1['stream']:
+            format = stream.get('stream_type')
             video_urls = []
-            for dt in data1['segs'][format]:
-                n = str(int(dt['no']))
+            for dt in stream['segs']:
+                n = str(int(dt['size']))
                 param = {
-                    'K': dt['k'],
+                    'K': dt['key'],
                     'hd': self.get_hd(format),
                     'myp': 0,
-                    'ts': dt['seconds'],
+                    'ts': dt['total_milliseconds_video'],
                     'ypp': 0,
                     'ctype': 12,
                     'ev': 1,
@@ -161,7 +168,8 @@ class YoukuIE(InfoExtractor):
             'hd2': '2',
             'hd3': '3',
             '3gp': '0',
-            '3gphd': '1'
+            '3gphd': '1',
+            'flvhd': '0'
         }
         return hd_id_dict[fm]
 
@@ -172,7 +180,8 @@ class YoukuIE(InfoExtractor):
             'hd2': 'flv',
             'hd3': 'flv',
             '3gp': 'flv',
-            '3gphd': 'mp4'
+            '3gphd': 'mp4',
+            'flvhd': 'flv'
         }
         return ext_dict[fm]
 
@@ -180,6 +189,7 @@ class YoukuIE(InfoExtractor):
         _dict = {
             '3gp': 'h6',
             '3gphd': 'h5',
+            'flvhd': 'h4',
             'flv': 'h4',
             'mp4': 'h3',
             'hd2': 'h2',
@@ -204,8 +214,9 @@ class YoukuIE(InfoExtractor):
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
+            jsonDumpIn = json.dumps(raw_data,indent = 1)
 
-            return raw_data['data']['security']
+            return raw_data['data']
 
 
         video_password = self._downloader.params.get('videopassword', None)
@@ -236,7 +247,8 @@ class YoukuIE(InfoExtractor):
                     msg += ': ' + error
                 raise ExtractorError(msg)
 
-        title = data1['title']
+        #title = data1['title']
+        title = data1['video']['title']
 
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(data1, data2)
@@ -248,10 +260,11 @@ class YoukuIE(InfoExtractor):
             'formats': [],
             # some formats are not available for all parts, we have to detect
             # which one has all
-        } for i in range(max(len(v) for v in data1['segs'].values()))]
-        for fm in data1['streamtypes']:
+        } for i in range(max(len(v) for v in data1['stream']))]
+        for stream in data1['stream']:
+            fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]
-            for video_url, seg, entry in zip(video_urls, data1['segs'][fm], entries):
+            for video_url, seg, entry in zip(video_urls, stream['segs'], entries):
                 entry['formats'].append({
                     'url': video_url,
                     'format_id': self.get_format_name(fm),

From ff51983e1563db08734e43a07d5671a517f99ef6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 18:52:03 +0800
Subject: [PATCH 2095/2145] [vevo] Handle videos without video_info (#7802)

---
 youtube_dl/extractor/vevo.py | 47 ++++++++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 1d834970c..38d382581 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -67,6 +67,17 @@ class VevoIE(InfoExtractor):
         'params': {
             'skip_download': 'true',
         }
+    }, {
+        'note': 'No video_info',
+        'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
+        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
+        'info_dict': {
+            'id': 'USUV71503000',
+            'ext': 'mp4',
+            'title': 'Till I Die - K Camp ft. T.I.',
+            'duration': 193,
+        },
+        'expected_warnings': ['HTTP Error 404'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
@@ -89,6 +100,9 @@ class VevoIE(InfoExtractor):
                 webpage, 'access token', fatal=False)
 
     def _formats_from_json(self, video_info):
+        if not video_info:
+            return []
+
         last_version = {'version': -1}
         for version in video_info['videoVersions']:
             # These are the HTTP downloads, other types are for different manifests
@@ -171,15 +185,24 @@ class VevoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        webpage = None
+
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(json_url, video_id)
-        video_info = response['video']
+        video_info = response['video'] or {}
 
-        if not video_info:
+        if not video_info and response.get('statusCode') != 909:
             if 'statusMessage' in response:
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusMessage']), expected=True)
             raise ExtractorError('Unable to extract videos')
 
+        if not video_info:
+            if url.startswith('vevo:'):
+                raise ExtractorError('Please specify full Vevo URL for downloading', expected=True)
+            webpage = self._download_webpage(url, video_id)
+
+        title = video_info.get('title') or self._og_search_title(webpage)
+
         formats = self._formats_from_json(video_info)
 
         is_explicit = video_info.get('isExplicit')
@@ -195,7 +218,7 @@ class VevoIE(InfoExtractor):
 
         # Download SMIL
         smil_blocks = sorted((
-            f for f in video_info['videoVersions']
+            f for f in video_info.get('videoVersions', [])
             if f['sourceType'] == 13),
             key=lambda f: f['version'])
         smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
@@ -213,17 +236,21 @@ class VevoIE(InfoExtractor):
                 formats.extend(self._formats_from_smil(smil_xml))
 
         self._sort_formats(formats)
-        timestamp_ms = int_or_none(self._search_regex(
+        timestamp = int_or_none(self._search_regex(
             r'/Date\((\d+)\)/',
-            video_info['launchDate'], 'launch date', fatal=False))
+            video_info['launchDate'], 'launch date', fatal=False),
+            scale=1000) if video_info else None
+
+        duration = video_info.get('duration') or int_or_none(
+            self._html_search_meta('video:duration', webpage))
 
         return {
             'id': video_id,
-            'title': video_info['title'],
+            'title': title,
             'formats': formats,
-            'thumbnail': video_info['imageUrl'],
-            'timestamp': timestamp_ms // 1000,
-            'uploader': video_info['mainArtists'][0]['artistName'],
-            'duration': video_info['duration'],
+            'thumbnail': video_info.get('imageUrl'),
+            'timestamp': timestamp,
+            'uploader': video_info['mainArtists'][0]['artistName'] if video_info else None,
+            'duration': duration,
             'age_limit': age_limit,
         }

From ad30dc1e20979842b6b81ee3f6d2364c78acf5f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 19:05:51 +0800
Subject: [PATCH 2096/2145] [vevo] Allow calling API without https

Not all proxies allow CONNECT
---
 youtube_dl/extractor/vevo.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 38d382581..3e3b25da0 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_etree_fromstring
+from ..compat import (
+    compat_etree_fromstring,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -162,14 +165,14 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _download_api_formats(self, video_id):
+    def _download_api_formats(self, video_id, video_url):
         if not self._oauth_token:
             self._downloader.report_warning(
                 'No oauth token available, skipping API HLS download')
             return []
 
-        api_url = 'https://apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
-            video_id, self._oauth_token)
+        api_url = compat_urlparse.urljoin(video_url, '//apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
+            video_id, self._oauth_token))
         api_data = self._download_json(
             api_url, video_id,
             note='Downloading HLS formats',
@@ -214,7 +217,7 @@ class VevoIE(InfoExtractor):
             age_limit = None
 
         # Download via HLS API
-        formats.extend(self._download_api_formats(video_id))
+        formats.extend(self._download_api_formats(video_id, url))
 
         # Download SMIL
         smil_blocks = sorted((

From b6afc225c89e3fc70a588e5822bbb2d86705bd08 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 19:16:51 +0800
Subject: [PATCH 2097/2145] [vevo] Use _download_smil to provide informative
 error messages

---
 youtube_dl/extractor/vevo.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 3e3b25da0..02dfd36f4 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -80,7 +80,7 @@ class VevoIE(InfoExtractor):
             'title': 'Till I Die - K Camp ft. T.I.',
             'duration': 193,
         },
-        'expected_warnings': ['HTTP Error 404'],
+        'expected_warnings': ['Unable to download SMIL file'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
@@ -130,9 +130,8 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _formats_from_smil(self, smil_xml):
+    def _formats_from_smil(self, smil_doc):
         formats = []
-        smil_doc = compat_etree_fromstring(smil_xml.encode('utf-8'))
         els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
@@ -233,10 +232,9 @@ class VevoIE(InfoExtractor):
             if smil_url_m is not None:
                 smil_url = smil_url_m
         if smil_url:
-            smil_xml = self._download_webpage(
-                smil_url, video_id, 'Downloading SMIL info', fatal=False)
-            if smil_xml:
-                formats.extend(self._formats_from_smil(smil_xml))
+            smil_doc = self._download_smil(smil_url, video_id, fatal=False)
+            if smil_doc:
+                formats.extend(self._formats_from_smil(smil_doc))
 
         self._sort_formats(formats)
         timestamp = int_or_none(self._search_regex(

From 98c3806b15adfcbd7a24fa67c8702c8a1adb0783 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Fri, 11 Dec 2015 19:18:14 +0800
Subject: [PATCH 2098/2145] fix some not important codesnips

---
 youtube_dl/extractor/youku.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 155286fc5..0e955db8a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,15 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import base64
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_cookiejar,
-    compat_cookies,
-    compat_urllib_request,
     compat_ord,
 )
 from ..utils import (
@@ -89,7 +85,7 @@ class YoukuIE(InfoExtractor):
         ).decode('ascii').split('_')
 
         # get oip
-        oip = data2['security']['ip']
+        oip = data1['security']['ip']
 
         # get fileid
         string_ls = list(
@@ -136,7 +132,8 @@ class YoukuIE(InfoExtractor):
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
-                n = str(int(dt['size']))
+                #n = str(int(dt['size']))
+                n = str(stream['segs'].index(dt))
                 param = {
                     'K': dt['key'],
                     'hd': self.get_hd(format),
@@ -177,6 +174,8 @@ class YoukuIE(InfoExtractor):
         ext_dict = {
             'flv': 'flv',
             'mp4': 'mp4',
+            'mp4hd': 'mp4',
+            'mp4hd2': 'mp4',
             'hd2': 'flv',
             'hd3': 'flv',
             '3gp': 'flv',
@@ -214,7 +213,6 @@ class YoukuIE(InfoExtractor):
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
-            jsonDumpIn = json.dumps(raw_data,indent = 1)
 
             return raw_data['data']
 
@@ -260,7 +258,7 @@ class YoukuIE(InfoExtractor):
             'formats': [],
             # some formats are not available for all parts, we have to detect
             # which one has all
-        } for i in range(max(len(v) for v in data1['stream']))]
+        } for i in range(max(len(v.get('segs')) for v in data1['stream']))]
         for stream in data1['stream']:
             fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]

From 5333842a1dd89320cfc0b15a483dba068e6c6af2 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Fri, 11 Dec 2015 20:08:14 +0800
Subject: [PATCH 2099/2145] According the blog and you-get fixed the issues
 #7627.

---
 youtube_dl/extractor/youku.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 0e955db8a..430a2bb8d 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -112,8 +112,13 @@ class YoukuIE(InfoExtractor):
             #fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
 
         def get_fileid(format, n):
+            number = hex(int(str(n), 10))[2:].upper()
+            if len(number) == 1:
+                number = '0' + number
+            streamfileids = fileid_dict[format]
+            fileid = streamfileids[0:8] + number + streamfileids[10:]
             #fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
-            fileid = fileid_dict[format]
+            #fileid = fileid_dict[format]
             return fileid
 
         # get ep
@@ -138,7 +143,7 @@ class YoukuIE(InfoExtractor):
                     'K': dt['key'],
                     'hd': self.get_hd(format),
                     'myp': 0,
-                    'ts': dt['total_milliseconds_video'],
+                    #'ts': dt['total_milliseconds_video'],
                     'ypp': 0,
                     'ctype': 12,
                     'ev': 1,
@@ -149,7 +154,7 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_' + str(int(n) + 1).zfill(2) + \
+                    '_00'+ \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)

From 4db43567e8833ca0eb89d694260e5640c414ca07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 20:28:44 +0600
Subject: [PATCH 2100/2145] [downloader/f4m] Decode manifest before fixing

---
 youtube_dl/downloader/f4m.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7819a91dc..aaf0c49c8 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -292,7 +292,7 @@ class F4mFD(FragmentFD):
         # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
         # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
         # and https://github.com/rg3/youtube-dl/issues/7823)
-        manifest = fix_xml_ampersands(urlh.read()).strip()
+        manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()
 
         doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)

From 2559b9d017985b1fc553028c4059b6c58eb78ea7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Dec 2015 17:31:33 +0100
Subject: [PATCH 2101/2145] [wdr] extract all formats(closes #7788)

---
 youtube_dl/extractor/wdr.py | 58 ++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index b46802306..ef096cbd2 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,8 +10,8 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
-    determine_ext,
     unified_strdate,
+    qualities,
 )
 
 
@@ -33,6 +33,7 @@ class WDRIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/themen/av/videomargaspiegelisttot101-videoplayer.html',
@@ -47,6 +48,7 @@ class WDRIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/themen/kultur/audioerlebtegeschichtenmargaspiegel100-audioplayer.html',
@@ -71,6 +73,7 @@ class WDRIE(InfoExtractor):
                 'upload_date': '20140717',
                 'is_live': False
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
@@ -83,10 +86,10 @@ class WDRIE(InfoExtractor):
             'url': 'http://www1.wdr.de/mediathek/video/livestream/index.html',
             'info_dict': {
                 'id': 'mdb-103364',
-                'title': 're:^WDR Fernsehen [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'title': 're:^WDR Fernsehen Live [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
                 'ext': 'flv',
-                'upload_date': '20150212',
+                'upload_date': '20150101',
                 'is_live': True
             },
             'params': {
@@ -150,25 +153,52 @@ class WDRIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
+        formats = []
+        preference = qualities(['S', 'M', 'L', 'XL'])
+
         if video_url.endswith('.f4m'):
-            video_url += '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
-            ext = 'flv'
+            f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         elif video_url.endswith('.smil'):
-            fmt = self._extract_smil_formats(video_url, page_id)[0]
-            video_url = fmt['url']
-            sep = '&' if '?' in video_url else '?'
-            video_url += sep
-            video_url += 'hdcore=3.3.0&plugin=aasp-3.3.0.99.43'
-            ext = fmt['ext']
+            smil_formats = self._extract_smil_formats(video_url, page_id, False, {
+                'hdcore': '3.3.0',
+                'plugin': 'aasp-3.3.0.99.43',
+            })
+            if smil_formats:
+                formats.extend(smil_formats)
         else:
-            ext = determine_ext(video_url)
+            formats.append({
+                'url': video_url,
+                'http_headers': {
+                    'User-Agent': 'mobile',
+                },
+            })
+
+        m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
+        if direct_urls:
+            for quality, video_url in direct_urls:
+                formats.append({
+                    'url': video_url,
+                    'preference': preference(quality),
+                    'http_headers': {
+                        'User-Agent': 'mobile',
+                    },
+                })
+
+        self._sort_formats(formats)
 
         description = self._html_search_meta('Description', webpage, 'description')
 
         return {
             'id': page_id,
-            'url': video_url,
-            'ext': ext,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From ab4bdc913fafc71217c1196602525e0382c4eea3 Mon Sep 17 00:00:00 2001
From: Muratcan Simsek <mail@muratcansimsek.com>
Date: Mon, 7 Dec 2015 00:15:19 +0200
Subject: [PATCH 2102/2145] [funimation] Add new extractor

Update funimation.py

Update funimation.py

Removed unnecessary lines.

Update funimation.py

Added thumbnail and description.

Filename improvement.

fixed TEST.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/funimation.py | 77 ++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 youtube_dl/extractor/funimation.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3db5cd6d9..3cd95ba01 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -206,6 +206,7 @@ from .francetv import (
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freevideo import FreeVideoIE
+from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
 from .gameinformer import GameInformerIE
 from .gamekings import GamekingsIE
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
new file mode 100644
index 000000000..773d80fe8
--- /dev/null
+++ b/youtube_dl/extractor/funimation.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    encode_dict,
+    sanitized_Request,
+    ExtractorError,
+    urlencode_postdata
+)
+import re
+
+class FunimationIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/.+[^ ]/videos/official/(?P<id>[^?]+)'
+
+    _TEST = {
+        'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
+        'info_dict': {
+            'id': 'AIRENG0001',
+            'title': 'Air - 1 - Breeze ',
+            'ext': 'mp4',
+            'thumbnail': 'http://www.funimation.com/admin/uploads/default/recap_thumbnails/7555590/home_spotlight/AIR0001.jpg',
+            'description': 'Travelling puppeteer Yukito arrives in a small town where he hopes to earn money through the magic of his puppets. When a young girl named Misuzu lures him to her home with the promise of food, his life changes forever. ',
+        }
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        login_url = 'http://www.funimation.com/login'
+        data = urlencode_postdata(encode_dict({
+            'loginForm2': 'loginform',
+            'email_field': username,
+            'password_field': password,
+        }))
+        login_request = sanitized_Request(login_url, data)
+        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        try:
+            login = self._download_webpage(login_request, login_url)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                raise ExtractorError('Funimation is not available in your region.', expected=True)
+            raise
+        if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
+                raise ExtractorError('Unable to login, wrong username or password.', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        try:
+            webpage = self._download_webpage(url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                raise ExtractorError('Funimation is not available in your region.', expected=True)
+            raise
+        if re.search(r'"sdUrl":"http', webpage) is None:
+                raise ExtractorError('You are not logged-in or the stream requires subscription.', expected=True)
+
+        m3u8 = self._search_regex(r'".+Url":"(.+?m3u8)"', webpage, 'm3u8') + self._search_regex(r'"authToken":"(.+?)"', webpage, 'm3u8')
+        formats = self._extract_m3u8_formats(m3u8.replace('\\', ''), video_id, ext='mp4', entry_protocol='m3u8_native')
+
+        video_show = self._search_regex(r'"artist":"(.+?)"', webpage, 'video_show')
+        video_track = self._search_regex(r'"videoNumber":"(\d+).0"', webpage, 'video_track')
+        video_title = self._search_regex(r'"title":"({0}.+?)"'.format(video_track), webpage, 'video_title')
+        video_id = self._search_regex(r'"FUNImationID":"(.+?)"', webpage, 'video_id')
+
+        return {
+            'id': video_id,
+            'title': video_show + ' - ' + video_title + ' ',
+            'formats': formats,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage)
+        }

From b4c299bad032c51736f52117fe8154751aecd6da Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:11:45 +0600
Subject: [PATCH 2103/2145] [funimation] PEP 8

---
 youtube_dl/extractor/funimation.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 773d80fe8..6f6aaf30f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -10,6 +10,7 @@ from ..utils import (
 )
 import re
 
+
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/.+[^ ]/videos/official/(?P<id>[^?]+)'
 
@@ -43,7 +44,7 @@ class FunimationIE(InfoExtractor):
                 raise ExtractorError('Funimation is not available in your region.', expected=True)
             raise
         if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
-                raise ExtractorError('Unable to login, wrong username or password.', expected=True)
+            raise ExtractorError('Unable to login, wrong username or password.', expected=True)
 
     def _real_initialize(self):
         self._login()

From 411e5b88c9d87f7deac35130d77f0c7243ff2a9d Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:13:49 +0600
Subject: [PATCH 2104/2145] [funimation] Fix login message

---
 youtube_dl/extractor/funimation.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 6f6aaf30f..8de28029f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -38,7 +38,8 @@ class FunimationIE(InfoExtractor):
         login_request = sanitized_Request(login_url, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         try:
-            login = self._download_webpage(login_request, login_url)
+            login = self._download_webpage(
+                login_request, None, 'Logging in as %s' % username)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 raise ExtractorError('Funimation is not available in your region.', expected=True)

From 40ca5b04f4845d3ab334d6cc9fb0272fe25df8b3 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:33:17 +0600
Subject: [PATCH 2105/2145] [funimation] Remove unnecessary login form field

---
 youtube_dl/extractor/funimation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8de28029f..74923c3cf 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -31,7 +31,6 @@ class FunimationIE(InfoExtractor):
             return
         login_url = 'http://www.funimation.com/login'
         data = urlencode_postdata(encode_dict({
-            'loginForm2': 'loginform',
             'email_field': username,
             'password_field': password,
         }))

From 59a4ff482a330ddf241ca4606aed66b5d029c055 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:34:30 +0600
Subject: [PATCH 2106/2145] [funimation] Real UA is required for login

---
 youtube_dl/extractor/funimation.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 74923c3cf..d0759f3b6 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -34,8 +34,10 @@ class FunimationIE(InfoExtractor):
             'email_field': username,
             'password_field': password,
         }))
-        login_request = sanitized_Request(login_url, data)
-        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_request = sanitized_Request(login_url, data, headers={
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+            'Content-Type': 'application/x-www-form-urlencoded'
+        })
         try:
             login = self._download_webpage(
                 login_request, None, 'Logging in as %s' % username)

From f542a3d26bfb70d59a0418387e508130e753b82e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 23:00:22 +0600
Subject: [PATCH 2107/2145] [funimation] Improve extraction (Closes #7775)

---
 youtube_dl/extractor/funimation.py | 137 ++++++++++++++++++++++-------
 1 file changed, 105 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index d0759f3b6..06a57779a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -1,18 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..compat import compat_urllib_request
 from ..utils import (
+    clean_html,
+    determine_ext,
     encode_dict,
     sanitized_Request,
     ExtractorError,
     urlencode_postdata
 )
-import re
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/.+[^ ]/videos/official/(?P<id>[^?]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^?]+)'
 
     _TEST = {
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
@@ -25,26 +29,31 @@ class FunimationIE(InfoExtractor):
         }
     }
 
+    def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):
+        HEADERS = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+        }
+        if isinstance(url_or_request, compat_urllib_request.Request):
+            for header, value in HEADERS.items():
+                url_or_request.add_header(header, value)
+        else:
+            url_or_request = sanitized_Request(url_or_request, headers=HEADERS)
+        response = super(FunimationIE, self)._download_webpage(url_or_request, video_id, note)
+        return response
+
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        login_url = 'http://www.funimation.com/login'
         data = urlencode_postdata(encode_dict({
             'email_field': username,
             'password_field': password,
         }))
-        login_request = sanitized_Request(login_url, data, headers={
-            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+        login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
             'Content-Type': 'application/x-www-form-urlencoded'
         })
-        try:
-            login = self._download_webpage(
-                login_request, None, 'Logging in as %s' % username)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                raise ExtractorError('Funimation is not available in your region.', expected=True)
-            raise
+        login = self._download_webpage(
+            login_request, None, 'Logging in as %s' % username)
         if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
             raise ExtractorError('Unable to login, wrong username or password.', expected=True)
 
@@ -52,29 +61,93 @@ class FunimationIE(InfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        try:
-            webpage = self._download_webpage(url, video_id)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                raise ExtractorError('Funimation is not available in your region.', expected=True)
-            raise
-        if re.search(r'"sdUrl":"http', webpage) is None:
-                raise ExtractorError('You are not logged-in or the stream requires subscription.', expected=True)
+        display_id = self._match_id(url)
 
-        m3u8 = self._search_regex(r'".+Url":"(.+?m3u8)"', webpage, 'm3u8') + self._search_regex(r'"authToken":"(.+?)"', webpage, 'm3u8')
-        formats = self._extract_m3u8_formats(m3u8.replace('\\', ''), video_id, ext='mp4', entry_protocol='m3u8_native')
+        webpage = self._download_webpage(url, display_id)
 
-        video_show = self._search_regex(r'"artist":"(.+?)"', webpage, 'video_show')
-        video_track = self._search_regex(r'"videoNumber":"(\d+).0"', webpage, 'video_track')
-        video_title = self._search_regex(r'"title":"({0}.+?)"'.format(video_track), webpage, 'video_title')
-        video_id = self._search_regex(r'"FUNImationID":"(.+?)"', webpage, 'video_id')
+        items = self._parse_json(
+            self._search_regex(
+                r'var\s+playersData\s*=\s*(\[.+?\]);\n',
+                webpage, 'players data'),
+            display_id)[0]['playlist'][0]['items']
+
+        item = next(item for item in items if item.get('itemAK') == display_id)
+
+        ERRORS_MAP = {
+            'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
+            'ERROR_MATURE_CONTENT_LOGGED_OUT': 'matureContentLoggedOut',
+            'ERROR_SUBSCRIPTION_LOGGED_OUT': 'subscriptionLoggedOut',
+            'ERROR_VIDEO_EXPIRED': 'videoExpired',
+            'ERROR_TERRITORY_UNAVAILABLE': 'territoryUnavailable',
+            'SVODBASIC_SUBSCRIPTION_IN_PLAYER': 'basicSubscription',
+            'SVODNON_SUBSCRIPTION_IN_PLAYER': 'nonSubscription',
+            'ERROR_PLAYER_NOT_RESPONDING': 'playerNotResponding',
+            'ERROR_UNABLE_TO_CONNECT_TO_CDN': 'unableToConnectToCDN',
+            'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
+        }
+
+        error_messages = {}
+        video_error_messages = self._search_regex(
+            r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
+            webpage, 'error messages', default=None)
+        if video_error_messages:
+            error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
+            if error_messages_json:
+                for _, error in error_messages_json.items():
+                    type_ = error.get('type')
+                    description = error.get('description')
+                    content = error.get('content')
+                    if type_ == 'text' and description and content:
+                        error_message = ERRORS_MAP.get(description)
+                        if error_message:
+                            error_messages[error_message] = content
+
+        errors = []
+        formats = []
+        for video in item['videoSet']:
+            auth_token = video.get('authToken')
+            if not auth_token:
+                continue
+            funimation_id = video.get('FUNImationID') or video.get('videoId')
+            if not auth_token.startswith('?'):
+                auth_token = '?%s' % auth_token
+            for quality in ('sd', 'hd', 'hd1080'):
+                format_url = video.get('%sUrl' % quality)
+                if not format_url:
+                    continue
+                if not format_url.startswith(('http', '//')):
+                    errors.append(format_url)
+                    continue
+                if determine_ext(format_url) == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(
+                        format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=funimation_id or 'hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                else:
+                    formats.append({
+                        'url': format_url + auth_token,
+                    })
+
+        if not formats and errors:
+            raise ExtractorError(
+                '%s returned error: %s'
+                % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
+                expected=True)
+
+        title = item['title']
+        artist = item.get('artist')
+        if artist:
+            title = '%s - %s' % (artist, title)
+        description = self._og_search_description(webpage) or item.get('description')
+        thumbnail = self._og_search_thumbnail(webpage) or item.get('posterUrl')
+        video_id = item.get('itemId') or display_id
 
         return {
             'id': video_id,
-            'title': video_show + ' - ' + video_title + ' ',
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage)
         }

From d357bbd375c9e30b882dfb8cd2cded7e19a9936c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 23:06:44 +0600
Subject: [PATCH 2108/2145] [funimation] Update test

---
 youtube_dl/extractor/funimation.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 06a57779a..e47fe9b73 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -21,12 +21,13 @@ class FunimationIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
-            'id': 'AIRENG0001',
-            'title': 'Air - 1 - Breeze ',
+            'id': '658',
+            'display_id': 'breeze',
             'ext': 'mp4',
-            'thumbnail': 'http://www.funimation.com/admin/uploads/default/recap_thumbnails/7555590/home_spotlight/AIR0001.jpg',
-            'description': 'Travelling puppeteer Yukito arrives in a small town where he hopes to earn money through the magic of his puppets. When a young girl named Misuzu lures him to her home with the promise of food, his life changes forever. ',
-        }
+            'title': 'Air - 1 - Breeze',
+            'description': 'md5:1769f43cd5fc130ace8fd87232207892',
+            'thumbnail': 're:https?://.*\.jpg',
+        },
     }
 
     def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):

From 9c163950da803b11f50bff4562b38effaa000cd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 23:20:10 +0600
Subject: [PATCH 2109/2145] [funimation] Improve _VALID_URL

---
 youtube_dl/extractor/funimation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index e47fe9b73..4f382ea6f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^?]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',

From b59623ef434cb7868faa4b5e6e0dd2a56449f369 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:38:58 +0600
Subject: [PATCH 2110/2145] [funimation] Use mobile webpage for workaround hulu
 error

---
 youtube_dl/extractor/funimation.py | 151 +++++++++++++++++------------
 1 file changed, 88 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 4f382ea6f..8af2e907c 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -18,7 +18,7 @@ from ..utils import (
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
             'id': '658',
@@ -28,19 +28,17 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:1769f43cd5fc130ace8fd87232207892',
             'thumbnail': 're:https?://.*\.jpg',
         },
-    }
-
-    def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):
-        HEADERS = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
-        }
-        if isinstance(url_or_request, compat_urllib_request.Request):
-            for header, value in HEADERS.items():
-                url_or_request.add_header(header, value)
-        else:
-            url_or_request = sanitized_Request(url_or_request, headers=HEADERS)
-        response = super(FunimationIE, self)._download_webpage(url_or_request, video_id, note)
-        return response
+    }, {
+        'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
+        'info_dict': {
+            'id': '31128',
+            'display_id': 'role-play',
+            'ext': 'mp4',
+            'title': '.hack//SIGN - 1 - Role Play',
+            'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
+            'thumbnail': 're:https?://.*\.jpg',
+        },
+    }]
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -51,6 +49,7 @@ class FunimationIE(InfoExtractor):
             'password_field': password,
         }))
         login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
             'Content-Type': 'application/x-www-form-urlencoded'
         })
         login = self._download_webpage(
@@ -64,15 +63,8 @@ class FunimationIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        items = self._parse_json(
-            self._search_regex(
-                r'var\s+playersData\s*=\s*(\[.+?\]);\n',
-                webpage, 'players data'),
-            display_id)[0]['playlist'][0]['items']
-
-        item = next(item for item in items if item.get('itemAK') == display_id)
+        errors = []
+        formats = []
 
         ERRORS_MAP = {
             'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
@@ -87,48 +79,79 @@ class FunimationIE(InfoExtractor):
             'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
         }
 
-        error_messages = {}
-        video_error_messages = self._search_regex(
-            r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
-            webpage, 'error messages', default=None)
-        if video_error_messages:
-            error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
-            if error_messages_json:
-                for _, error in error_messages_json.items():
-                    type_ = error.get('type')
-                    description = error.get('description')
-                    content = error.get('content')
-                    if type_ == 'text' and description and content:
-                        error_message = ERRORS_MAP.get(description)
-                        if error_message:
-                            error_messages[error_message] = content
+        USER_AGENTS = (
+            # PC UA is served with m3u8 that provides some bonus lower quality formats
+            ('pc', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'),
+            # Mobile UA allows to extract direct links and also does not fail when
+            # PC UA fails with hulu error (e.g.
+            # http://www.funimation.com/shows/hacksign/videos/official/role-play)
+            ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
+        )
 
-        errors = []
-        formats = []
-        for video in item['videoSet']:
-            auth_token = video.get('authToken')
-            if not auth_token:
-                continue
-            funimation_id = video.get('FUNImationID') or video.get('videoId')
-            if not auth_token.startswith('?'):
-                auth_token = '?%s' % auth_token
-            for quality in ('sd', 'hd', 'hd1080'):
-                format_url = video.get('%sUrl' % quality)
-                if not format_url:
+        for kind, user_agent in USER_AGENTS:
+            request = sanitized_Request(url)
+            request.add_header('User-Agent', user_agent)
+            webpage = self._download_webpage(
+                request, display_id, 'Downloading %s webpage' % kind)
+
+            items = self._parse_json(
+                self._search_regex(
+                    r'var\s+playersData\s*=\s*(\[.+?\]);\n',
+                    webpage, 'players data'),
+                display_id)[0]['playlist'][0]['items']
+
+            item = next(item for item in items if item.get('itemAK') == display_id)
+
+            error_messages = {}
+            video_error_messages = self._search_regex(
+                r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
+                webpage, 'error messages', default=None)
+            if video_error_messages:
+                error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
+                if error_messages_json:
+                    for _, error in error_messages_json.items():
+                        type_ = error.get('type')
+                        description = error.get('description')
+                        content = error.get('content')
+                        if type_ == 'text' and description and content:
+                            error_message = ERRORS_MAP.get(description)
+                            if error_message:
+                                error_messages[error_message] = content
+
+            for video in item.get('videoSet', []):
+                auth_token = video.get('authToken')
+                if not auth_token:
                     continue
-                if not format_url.startswith(('http', '//')):
-                    errors.append(format_url)
-                    continue
-                if determine_ext(format_url) == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
-                        format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                        m3u8_id=funimation_id or 'hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
-                else:
-                    formats.append({
-                        'url': format_url + auth_token,
-                    })
+                funimation_id = video.get('FUNImationID') or video.get('videoId')
+                preference = 1 if video.get('languageMode') == 'dub' else 0
+                if not auth_token.startswith('?'):
+                    auth_token = '?%s' % auth_token
+                for quality in ('sd', 'hd', 'hd1080'):
+                    format_url = video.get('%sUrl' % quality)
+                    if not format_url:
+                        continue
+                    if not format_url.startswith(('http', '//')):
+                        errors.append(format_url)
+                        continue
+                    if determine_ext(format_url) == 'm3u8':
+                        m3u8_formats = self._extract_m3u8_formats(
+                            format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
+                            preference=preference, m3u8_id=funimation_id or 'hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
+                    else:
+                        f = {
+                            'url': format_url + auth_token,
+                            'format_id': funimation_id,
+                            'preference': preference,
+                        }
+                        mobj = re.search(r'(?P<height>\d+)-(?P<tbr>\d+)[Kk]', format_url)
+                        if mobj:
+                            f.update({
+                                'height': int(mobj.group('height')),
+                                'tbr': int(mobj.group('tbr')),
+                            })
+                        formats.append(f)
 
         if not formats and errors:
             raise ExtractorError(
@@ -136,6 +159,8 @@ class FunimationIE(InfoExtractor):
                 % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
                 expected=True)
 
+        self._sort_formats(formats)
+
         title = item['title']
         artist = item.get('artist')
         if artist:

From b323a3cbff32df7188fc4fbb899dcc1c44e5d064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:39:44 +0600
Subject: [PATCH 2111/2145] [funimation] Remove unused import

---
 youtube_dl/extractor/funimation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8af2e907c..35328997b 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     clean_html,
     determine_ext,

From b091529a3ccfdcc638939b71f53708f944450df1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:43:03 +0600
Subject: [PATCH 2112/2145] [funimation] Extend _VALID_URL to match promotional
 videos

---
 youtube_dl/extractor/funimation.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 35328997b..0b4279612 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
@@ -37,6 +37,9 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
             'thumbnail': 're:https?://.*\.jpg',
         },
+    }, {
+        'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
+        'only_matching': True,
     }]
 
     def _login(self):

From f208e52a76231b112c6a9d64e2fd32712e7812e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:48:09 +0600
Subject: [PATCH 2113/2145] [funimation] Fix promotional videos extraction

---
 youtube_dl/extractor/funimation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 0b4279612..5322e4e93 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -96,12 +96,13 @@ class FunimationIE(InfoExtractor):
             webpage = self._download_webpage(
                 request, display_id, 'Downloading %s webpage' % kind)
 
-            items = self._parse_json(
+            playlist = self._parse_json(
                 self._search_regex(
                     r'var\s+playersData\s*=\s*(\[.+?\]);\n',
                     webpage, 'players data'),
-                display_id)[0]['playlist'][0]['items']
+                display_id)[0]['playlist']
 
+            items = next(item['items'] for item in playlist if item.get('items'))
             item = next(item for item in items if item.get('itemAK') == display_id)
 
             error_messages = {}

From 0b1bb1ac3ac25dd4d248c71fec16b2a995ecb935 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:52:00 +0600
Subject: [PATCH 2114/2145] [funimation] Add test for promotional video

---
 youtube_dl/extractor/funimation.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 5322e4e93..35125b631 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -39,7 +39,14 @@ class FunimationIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
-        'only_matching': True,
+        'info_dict': {
+            'id': '9635',
+            'display_id': 'broadcast-dub-preview',
+            'ext': 'mp4',
+            'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
+            'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
+            'thumbnail': 're:https?://.*\.(?:jpg|png)',
+        },
     }]
 
     def _login(self):

From f377f44dae78ff6837d9301e7a086dfb1e601df2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 01:02:54 +0600
Subject: [PATCH 2115/2145] [funimation] Improve extraction

---
 youtube_dl/extractor/funimation.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 35125b631..8a65343c5 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -8,6 +8,7 @@ from ..utils import (
     clean_html,
     determine_ext,
     encode_dict,
+    int_or_none,
     sanitized_Request,
     ExtractorError,
     urlencode_postdata
@@ -136,7 +137,7 @@ class FunimationIE(InfoExtractor):
                 preference = 1 if video.get('languageMode') == 'dub' else 0
                 if not auth_token.startswith('?'):
                     auth_token = '?%s' % auth_token
-                for quality in ('sd', 'hd', 'hd1080'):
+                for quality, height in (('sd', 480), ('hd', 720), ('hd1080', 1080)):
                     format_url = video.get('%sUrl' % quality)
                     if not format_url:
                         continue
@@ -146,22 +147,19 @@ class FunimationIE(InfoExtractor):
                     if determine_ext(format_url) == 'm3u8':
                         m3u8_formats = self._extract_m3u8_formats(
                             format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id=funimation_id or 'hls', fatal=False)
+                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False)
                         if m3u8_formats:
                             formats.extend(m3u8_formats)
                     else:
-                        f = {
+                        tbr = int_or_none(self._search_regex(
+                            r'-(\d+)[Kk]', format_url, 'tbr', default=None))
+                        formats.append({
                             'url': format_url + auth_token,
-                            'format_id': funimation_id,
+                            'format_id': '%s-http-%dp' % (funimation_id, height),
+                            'height': height,
+                            'tbr': tbr,
                             'preference': preference,
-                        }
-                        mobj = re.search(r'(?P<height>\d+)-(?P<tbr>\d+)[Kk]', format_url)
-                        if mobj:
-                            f.update({
-                                'height': int(mobj.group('height')),
-                                'tbr': int(mobj.group('tbr')),
-                            })
-                        formats.append(f)
+                        })
 
         if not formats and errors:
             raise ExtractorError(

From d63cfc3f0faa7d52900f2f1959e8ad81bcef2775 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 02:52:20 +0600
Subject: [PATCH 2116/2145] [beeg] API v5 (Closes #7846)

---
 youtube_dl/extractor/beeg.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index d151d38c9..c8d921daf 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,15 +34,29 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v4/video/%s' % video_id, video_id)
+            'http://beeg.com/api/v5/video/%s' % video_id, video_id)
+
+        def split(o, e):
+            def cut(s, x):
+                n.append(s[:x])
+                return s[x:]
+            n = []
+            r = len(o) % e
+            if r > 0:
+                o = cut(o, r)
+            while len(o) > e:
+                o = cut(o, e)
+            n.append(o)
+            return n
 
         def decrypt_key(key):
-            # Reverse engineered from http://static.beeg.com/cpl/1067.js
-            a = '8RPUUCS35ZWp3ADnKcSmpH71ZusrROo'
+            # Reverse engineered from http://static.beeg.com/cpl/1105.js
+            a = '5ShMcIQlssOd7zChAIOlmeTZDaUxULbJRnywYaiB'
             e = compat_urllib_parse_unquote(key)
-            return ''.join([
-                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 25)
+            o = ''.join([
+                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)
                 for n in range(len(e))])
+            return ''.join(split(o, 3)[::-1])
 
         def decrypt_url(encrypted_url):
             encrypted_url = self._proto_relative_url(

From 8696a7fd1396e051a3762ff241d1b4e157b7b08a Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Sat, 12 Dec 2015 10:44:21 +0800
Subject: [PATCH 2117/2145] fix the keyerror(mp4hd), todo support download the
 video protected by password

---
 youtube_dl/extractor/youku.py | 35 +++++++++++++----------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 430a2bb8d..9fd0e1e23 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -90,26 +91,12 @@ class YoukuIE(InfoExtractor):
         # get fileid
         string_ls = list(
             'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
-        shuffled_string_ls = []
-        seed = data1.get('seed')
-        seed = 0
-        N = len(string_ls)
-        for ii in range(N):
-            seed = (seed * 0xd3 + 0x754f) % 0x10000
-            idx = seed * len(string_ls) // 0x10000
-            shuffled_string_ls.append(string_ls[idx])
-            del string_ls[idx]
 
         fileid_dict = {}
         for stream in data1['stream']:
             format = stream.get('stream_type')
-            #streamfileid = [
-            #    int(i) for i in data1['stream']['streamfileids']#[format].strip('*').split('*')]
-            #fileid = ''.join(
-            #    [shuffled_string_ls[i] for i in streamfileid])
             fileid = stream['stream_fileid']
             fileid_dict[format] = fileid
-            #fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
 
         def get_fileid(format, n):
             number = hex(int(str(n), 10))[2:].upper()
@@ -117,8 +104,6 @@ class YoukuIE(InfoExtractor):
                 number = '0' + number
             streamfileids = fileid_dict[format]
             fileid = streamfileids[0:8] + number + streamfileids[10:]
-            #fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
-            #fileid = fileid_dict[format]
             return fileid
 
         # get ep
@@ -171,7 +156,9 @@ class YoukuIE(InfoExtractor):
             'hd3': '3',
             '3gp': '0',
             '3gphd': '1',
-            'flvhd': '0'
+            'flvhd': '0',
+            'mp4hd': '1',
+            'mp4hd2': '1'
         }
         return hd_id_dict[fm]
 
@@ -180,7 +167,8 @@ class YoukuIE(InfoExtractor):
             'flv': 'flv',
             'mp4': 'mp4',
             'mp4hd': 'mp4',
-            'mp4hd2': 'mp4',
+            'mp4hd2': 'flv',
+            'mp4hd3': 'flv',
             'hd2': 'flv',
             'hd3': 'flv',
             '3gp': 'flv',
@@ -197,7 +185,10 @@ class YoukuIE(InfoExtractor):
             'flv': 'h4',
             'mp4': 'h3',
             'hd2': 'h2',
-            'hd3': 'h1'
+            'hd3': 'h1',
+            'mp4hd': 'h3',
+            'mp4hd3': 'h4',
+            'mp4hd2': 'h4'
         }
         return _dict[fm]
 
@@ -218,6 +209,7 @@ class YoukuIE(InfoExtractor):
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
+            js = json.dumps(raw_data)
 
             return raw_data['data']
 
@@ -225,7 +217,6 @@ class YoukuIE(InfoExtractor):
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
-        #basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
         basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
         if video_password:
             basic_data_url += '?password=%s' % video_password
@@ -234,7 +225,6 @@ class YoukuIE(InfoExtractor):
             basic_data_url,
             'Downloading JSON metadata 1')
         data2 = retrieve_data(
-            #'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
             "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id,
             'Downloading JSON metadata 2')
 
@@ -250,9 +240,10 @@ class YoukuIE(InfoExtractor):
                     msg += ': ' + error
                 raise ExtractorError(msg)
 
-        #title = data1['title']
+        #get video title
         title = data1['video']['title']
 
+
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(data1, data2)
 

From 14c17cafa18292112e5ea09e23f8598892ab3a0b Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Sat, 12 Dec 2015 11:21:44 +0800
Subject: [PATCH 2118/2145] add support to video protected by password

---
 youtube_dl/extractor/youku.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 9fd0e1e23..f1bb482c3 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -219,25 +219,25 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
         if video_password:
-            basic_data_url += '?password=%s' % video_password
+            basic_data_url += '&pwd=%s' % video_password
 
         data1 = retrieve_data(
             basic_data_url,
             'Downloading JSON metadata 1')
         data2 = retrieve_data(
-            "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id,
+            basic_data_url,
             'Downloading JSON metadata 2')
 
-        error_code = data1.get('error_code')
-        if error_code:
-            error = data1.get('error')
-            if error is not None and '因版权原因无法观看此视频' in error:
+        error = data1.get('error')
+        if error:
+            error_note = error.get('note')
+            if error_note is not None and '因版权原因无法观看此视频' in error_note:
                 raise ExtractorError(
                     'Youku said: Sorry, this video is available in China only', expected=True)
             else:
-                msg = 'Youku server reported error %i' % error_code
+                msg = 'Youku server reported error %i' % error.get('code')
                 if error is not None:
-                    msg += ': ' + error
+                    msg += ': ' + error_note
                 raise ExtractorError(msg)
 
         #get video title

From 7e37c394858ac215b02d1c4ea425d20a17bf4655 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Sat, 12 Dec 2015 11:26:15 +0800
Subject: [PATCH 2119/2145] merge data1 and data2

---
 youtube_dl/extractor/youku.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index f1bb482c3..5110efcb4 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -64,7 +64,7 @@ class YoukuIE(InfoExtractor):
         },
     }]
 
-    def construct_video_urls(self, data1, data2):
+    def construct_video_urls(self, data):
         # get sid, token
         def yk_t(s1, s2):
             ls = list(range(256))
@@ -82,18 +82,18 @@ class YoukuIE(InfoExtractor):
             return bytes(s)
 
         sid, token = yk_t(
-            b'becaf9be', base64.b64decode(data2['security']['encrypt_string'].encode('ascii'))
+            b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii'))
         ).decode('ascii').split('_')
 
         # get oip
-        oip = data1['security']['ip']
+        oip = data['security']['ip']
 
         # get fileid
         string_ls = list(
             'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
 
         fileid_dict = {}
-        for stream in data1['stream']:
+        for stream in data['stream']:
             format = stream.get('stream_type')
             fileid = stream['stream_fileid']
             fileid_dict[format] = fileid
@@ -118,7 +118,7 @@ class YoukuIE(InfoExtractor):
 
         # generate video_urls
         video_urls_dict = {}
-        for stream in data1['stream']:
+        for stream in data['stream']:
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
@@ -221,14 +221,11 @@ class YoukuIE(InfoExtractor):
         if video_password:
             basic_data_url += '&pwd=%s' % video_password
 
-        data1 = retrieve_data(
+        data = retrieve_data(
             basic_data_url,
             'Downloading JSON metadata 1')
-        data2 = retrieve_data(
-            basic_data_url,
-            'Downloading JSON metadata 2')
 
-        error = data1.get('error')
+        error = data.get('error')
         if error:
             error_note = error.get('note')
             if error_note is not None and '因版权原因无法观看此视频' in error_note:
@@ -241,11 +238,11 @@ class YoukuIE(InfoExtractor):
                 raise ExtractorError(msg)
 
         #get video title
-        title = data1['video']['title']
+        title = data['video']['title']
 
 
         # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data1, data2)
+        video_urls_dict = self.construct_video_urls(data)
 
         # construct info
         entries = [{
@@ -254,8 +251,8 @@ class YoukuIE(InfoExtractor):
             'formats': [],
             # some formats are not available for all parts, we have to detect
             # which one has all
-        } for i in range(max(len(v.get('segs')) for v in data1['stream']))]
-        for stream in data1['stream']:
+        } for i in range(max(len(v.get('segs')) for v in data['stream']))]
+        for stream in data['stream']:
             fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]
             for video_url, seg, entry in zip(video_urls, stream['segs'], entries):

From 8773f3158fb45fcadadc0b5e8e9dfd5c9915fa39 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 23:59:25 +0800
Subject: [PATCH 2120/2145] [safari] Use postdata_urlencode (#7465)

---
 youtube_dl/extractor/safari.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 919704261..7de7b7273 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -6,12 +6,12 @@ import re
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
 
-from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     sanitized_Request,
     smuggle_url,
     std_headers,
+    urlencode_postdata,
 )
 
 
@@ -57,7 +57,7 @@ class SafariBaseIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
+            self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From f133fd326bf691db389acd8871b5fc7c6d4bc978 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 14:41:53 +0800
Subject: [PATCH 2121/2145] [youku] Cleanup and PEP8

---
 youtube_dl/extractor/youku.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 5110efcb4..bec29196a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import base64
-import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -88,10 +87,6 @@ class YoukuIE(InfoExtractor):
         # get oip
         oip = data['security']['ip']
 
-        # get fileid
-        string_ls = list(
-            'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
-
         fileid_dict = {}
         for stream in data['stream']:
             format = stream.get('stream_type')
@@ -122,13 +117,11 @@ class YoukuIE(InfoExtractor):
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
-                #n = str(int(dt['size']))
                 n = str(stream['segs'].index(dt))
                 param = {
                     'K': dt['key'],
                     'hd': self.get_hd(format),
                     'myp': 0,
-                    #'ts': dt['total_milliseconds_video'],
                     'ypp': 0,
                     'ctype': 12,
                     'ev': 1,
@@ -139,7 +132,7 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_00'+ \
+                    '_00' + \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)
@@ -196,24 +189,20 @@ class YoukuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         def retrieve_data(req_url, note):
-            
-
             headers = {
-                    'Referer': req_url,
-                }
-            self._set_cookie('youku.com','xreferrer','http://www.youku.com')
-            req = sanitized_Request(req_url,headers=headers)
+                'Referer': req_url,
+            }
+            self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
+            req = sanitized_Request(req_url, headers=headers)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
-            js = json.dumps(raw_data)
 
             return raw_data['data']
 
-
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
@@ -237,10 +226,9 @@ class YoukuIE(InfoExtractor):
                     msg += ': ' + error_note
                 raise ExtractorError(msg)
 
-        #get video title
+        # get video title
         title = data['video']['title']
 
-
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(data)
 

From c683454e7e35a67a901526071136f8932adab3ea Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 14:48:46 +0800
Subject: [PATCH 2122/2145] [youku] MD5 is unstable

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index bec29196a..2a65ae627 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -25,8 +25,8 @@ class YoukuIE(InfoExtractor):
     '''
 
     _TESTS = [{
+        # MD5 is unstable
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
-        'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
             'id': 'XMTc1ODE5Njcy_part1',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',

From 4d77550cf0d968344a6762c90496ffbe7528cbe7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 14:57:14 +0800
Subject: [PATCH 2123/2145] [youku] Fix tests

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 2a65ae627..b6129ecfa 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -42,6 +42,7 @@ class YoukuIE(InfoExtractor):
             'title': '武媚娘传奇 85',
         },
         'playlist_count': 11,
+        'skip': 'Available in China only',
     }, {
         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
         'info_dict': {
@@ -49,7 +50,6 @@ class YoukuIE(InfoExtractor):
             'title': '花千骨 04',
         },
         'playlist_count': 13,
-        'skip': 'Available in China only',
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
         'note': 'Video protected with password',

From ade23409717b45c14fda8446389f6dd8c2ca3bae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:19:14 +0800
Subject: [PATCH 2124/2145] [youku] Simplify

---
 youtube_dl/extractor/youku.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index b6129ecfa..1b1878f37 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -210,9 +210,7 @@ class YoukuIE(InfoExtractor):
         if video_password:
             basic_data_url += '&pwd=%s' % video_password
 
-        data = retrieve_data(
-            basic_data_url,
-            'Downloading JSON metadata 1')
+        data = retrieve_data(basic_data_url, 'Downloading JSON metadata')
 
         error = data.get('error')
         if error:

From dbb7d7e26cfba1cb8c9f78164757f594a2a50fd1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:24:58 +0800
Subject: [PATCH 2125/2145] [youku] Reorder format items

---
 youtube_dl/extractor/youku.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 1b1878f37..b6cc4bb4c 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -143,30 +143,30 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv': '0',
-            'mp4': '1',
-            'hd2': '2',
-            'hd3': '3',
             '3gp': '0',
             '3gphd': '1',
+            'flv': '0',
             'flvhd': '0',
+            'mp4': '1',
             'mp4hd': '1',
-            'mp4hd2': '1'
+            'mp4hd2': '1',
+            'hd2': '2',
+            'hd3': '3',
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
+            '3gp': 'flv',
+            '3gphd': 'mp4',
             'flv': 'flv',
+            'flvhd': 'flv',
             'mp4': 'mp4',
             'mp4hd': 'mp4',
             'mp4hd2': 'flv',
             'mp4hd3': 'flv',
             'hd2': 'flv',
             'hd3': 'flv',
-            '3gp': 'flv',
-            '3gphd': 'mp4',
-            'flvhd': 'flv'
         }
         return ext_dict[fm]
 
@@ -174,14 +174,14 @@ class YoukuIE(InfoExtractor):
         _dict = {
             '3gp': 'h6',
             '3gphd': 'h5',
-            'flvhd': 'h4',
             'flv': 'h4',
+            'flvhd': 'h4',
             'mp4': 'h3',
+            'mp4hd': 'h3',
+            'mp4hd2': 'h4',
+            'mp4hd3': 'h4',
             'hd2': 'h2',
             'hd3': 'h1',
-            'mp4hd': 'h3',
-            'mp4hd3': 'h4',
-            'mp4hd2': 'h4'
         }
         return _dict[fm]
 

From 174964a7bc4b8d759a4019e366520c9121fcbf15 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:34:40 +0800
Subject: [PATCH 2126/2145] Credit @Celthi for fixing Youku extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cdb56de3b..e0fb098e6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -147,3 +147,4 @@ Qijiang Fan
 Rémy Léone
 Marco Ferragina
 reiv
+Evan Lu

From deb1e8d20e8680d8e217e2d376e05836541a46ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:49:19 +0800
Subject: [PATCH 2127/2145] [youku] Put the missing item to get_hd

---
 youtube_dl/extractor/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index b6cc4bb4c..d33caa79e 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -150,6 +150,7 @@ class YoukuIE(InfoExtractor):
             'mp4': '1',
             'mp4hd': '1',
             'mp4hd2': '1',
+            'mp4hd3': '1',
             'hd2': '2',
             'hd3': '3',
         }

From c03943f39440d80e82239c82958b5d2f48f25410 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 15:19:23 +0600
Subject: [PATCH 2128/2145] Credit @Slyneth for funimation (#7775)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e0fb098e6..ce350e96c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -147,4 +147,5 @@ Qijiang Fan
 Rémy Léone
 Marco Ferragina
 reiv
+Muratcan Simsek
 Evan Lu

From 0014ffa829ad0d39d7231f664ea4436d2edd5c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Dec 2015 07:17:42 +0600
Subject: [PATCH 2129/2145] [funimation] Improve login

---
 youtube_dl/extractor/funimation.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8a65343c5..d1a95d87f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
@@ -18,6 +16,8 @@ from ..utils import (
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'
 
+    _NETRC_MACHINE = 'funimation'
+
     _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
@@ -62,10 +62,16 @@ class FunimationIE(InfoExtractor):
             'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
             'Content-Type': 'application/x-www-form-urlencoded'
         })
-        login = self._download_webpage(
+        login_page = self._download_webpage(
             login_request, None, 'Logging in as %s' % username)
-        if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
-            raise ExtractorError('Unable to login, wrong username or password.', expected=True)
+        if any(p in login_page for p in ('funimation.com/logout', '>Log Out<')):
+            return
+        error = self._html_search_regex(
+            r'(?s)<div[^>]+id=["\']errorMessages["\'][^>]*>(.+?)</div>',
+            login_page, 'error messages', default=None)
+        if error:
+            raise ExtractorError('Unable to login: %s' % error, expected=True)
+        raise ExtractorError('Unable to log in')
 
     def _real_initialize(self):
         self._login()

From 47c7f3d99504325c43651b781376ea470e7e28bd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 15:33:11 +0800
Subject: [PATCH 2130/2145] [novamov] Fix filekey extraction (closes #7764)

---
 youtube_dl/extractor/novamov.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 837c91559..0c4ff90f2 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -23,9 +23,10 @@ class NovaMovIE(InfoExtractor):
     _HOST = 'www.novamov.com'
 
     _FILE_DELETED_REGEX = r'This file no longer exists on our servers!</h2>'
-    _FILEKEY_REGEX = r'flashvars\.filekey="(?P<filekey>[^"]+)";'
+    _FILEKEY_REGEX = r'flashvars\.filekey=(?P<filekey>"?[^"]+"?);'
     _TITLE_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>'
     _DESCRIPTION_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>'
+    _URL_TEMPLATE = 'http://%s/video/%s'
 
     _TEST = {
         'url': 'http://www.novamov.com/video/4rurhn9x446jj',
@@ -42,7 +43,7 @@ class NovaMovIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        url = 'http://%s/video/%s' % (self._HOST, video_id)
+        url = self._URL_TEMPLATE % (self._HOST, video_id)
 
         webpage = self._download_webpage(
             url, video_id, 'Downloading video page')
@@ -51,8 +52,13 @@ class NovaMovIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         def extract_filekey(default=NO_DEFAULT):
-            return self._search_regex(
+            filekey = self._search_regex(
                 self._FILEKEY_REGEX, webpage, 'filekey', default=default)
+            if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
+                return self._search_regex(
+                    r'var\s*%s\s*=\s*"([^"]+)"', webpage, 'filekey', default=default)
+            else:
+                return filekey
 
         filekey = extract_filekey(default=None)
 
@@ -127,7 +133,6 @@ class NowVideoIE(NovaMovIE):
     _HOST = 'www.nowvideo.to'
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
     _TITLE_REGEX = r'<h4>([^<]+)</h4>'
     _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
 
@@ -153,6 +158,7 @@ class VideoWeedIE(NovaMovIE):
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
     _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
+    _URL_TEMPLATE = 'http://%s/file/%s'
 
     _TEST = {
         'url': 'http://www.videoweed.es/file/b42178afbea14',

From b5424acdb9983dbb761aed2f014dadeeac355930 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 15:37:52 +0800
Subject: [PATCH 2131/2145] [novamov] Improve existence checking

---
 youtube_dl/extractor/novamov.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 0c4ff90f2..a2a527ac3 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -40,6 +40,10 @@ class NovaMovIE(InfoExtractor):
         'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
     }
 
+    def _check_existence(self, webpage, video_id):
+        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -48,8 +52,7 @@ class NovaMovIE(InfoExtractor):
         webpage = self._download_webpage(
             url, video_id, 'Downloading video page')
 
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+        self._check_existence(webpage, video_id)
 
         def extract_filekey(default=NO_DEFAULT):
             filekey = self._search_regex(
@@ -75,6 +78,7 @@ class NovaMovIE(InfoExtractor):
             request.add_header('Referer', post_url)
             webpage = self._download_webpage(
                 request, video_id, 'Downloading continue to the video page')
+            self._check_existence(webpage, video_id)
 
         filekey = extract_filekey()
 

From f6abca506ef5274cfab151027bb3624d7b0e6a6b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 15:42:34 +0800
Subject: [PATCH 2132/2145] [nowvideo] Skip deleted test case

---
 youtube_dl/extractor/novamov.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index a2a527ac3..422fb993d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -148,7 +148,8 @@ class NowVideoIE(NovaMovIE):
             'ext': 'flv',
             'title': 'youtubedl test video _BaW_jenozKc.mp4',
             'description': 'Description',
-        }
+        },
+        'skip': 'Video 0mw0yow7b6dxa does not exist',
     }
 
 

From 22d07ba4e4fc1237cc015c971370ea9faf900610 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 17:29:27 +0800
Subject: [PATCH 2133/2145] [infoq] Fix extraction for HTTP URLs (closes #7739)

---
 youtube_dl/extractor/infoq.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 71cfd12c5..89bc3039c 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -3,10 +3,7 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urlparse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class InfoQIE(InfoExtractor):
@@ -45,9 +42,11 @@ class InfoQIE(InfoExtractor):
         video_filename = playpath.split('/')[-1]
         video_id, extension = video_filename.split('.')
 
-        http_base = self._search_regex(
-            r'EXPRESSINSTALL_SWF\s*=\s*[^"]*"((?:https?:)?//[^/"]+/)', webpage,
-            'HTTP base URL')
+        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
+
+        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
+        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
+        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
 
         formats = [{
             'format_id': 'rtmp',
@@ -56,7 +55,11 @@ class InfoQIE(InfoExtractor):
             'play_path': playpath,
         }, {
             'format_id': 'http',
-            'url': compat_urlparse.urljoin(url, http_base) + real_id,
+            'url': http_video_url,
+            'http_headers': {
+                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
+                    policy, signature, key_pair_id),
+            },
         }]
         self._sort_formats(formats)
 

From f3711edcf133bb621f6f58efa6753965573d6629 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 13 Dec 2015 10:52:59 +0100
Subject: [PATCH 2134/2145] release 2015.12.13

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3bb3abb25..8253335e3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -188,6 +188,7 @@
  - **Freesound**
  - **freespeech.org**
  - **FreeVideo**
+ - **Funimation**
  - **FunnyOrDie**
  - **GameInformer**
  - **Gamekings**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6cd748b48..20b44b94d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.10'
+__version__ = '2015.12.13'

From 02a63fadc357190ea116690a50610f04b61da976 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 19:15:45 +0800
Subject: [PATCH 2135/2145] [infoq] Refactor and support the Chinese version

Closes #7576
---
 youtube_dl/extractor/infoq.py | 108 ++++++++++++++++++++++++----------
 1 file changed, 76 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 89bc3039c..016af2084 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -1,9 +1,15 @@
+# coding: utf-8
+
 from __future__ import unicode_literals
 
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_parse_qs,
+)
+from ..utils import determine_ext
 
 
 class InfoQIE(InfoExtractor):
@@ -13,7 +19,7 @@ class InfoQIE(InfoExtractor):
         'url': 'http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things',
         'md5': 'b5ca0e0a8c1fed93b0e65e48e462f9a2',
         'info_dict': {
-            'id': '12-jan-pythonthings',
+            'id': 'A-Few-of-My-Favorite-Python-Things',
             'ext': 'mp4',
             'description': 'Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.',
             'title': 'A Few of My Favorite [Python] Things',
@@ -21,8 +27,71 @@ class InfoQIE(InfoExtractor):
     }, {
         'url': 'http://www.infoq.com/fr/presentations/changez-avis-sur-javascript',
         'only_matching': True,
+    }, {
+        'url': 'http://www.infoq.com/cn/presentations/openstack-continued-delivery',
+        'md5': '4918d0cca1497f2244572caf626687ef',
+        'info_dict': {
+            'id': 'openstack-continued-delivery',
+            'title': 'OpenStack持续交付之路',
+            'ext': 'flv',
+            'description': 'md5:308d981fb28fa42f49f9568322c683ff',
+        },
     }]
 
+    def _extract_bokecc_videos(self, webpage, video_id):
+        # TODO: bokecc.com is a Chinese video cloud platform
+        # It should have an independent extractor but I don't have other
+        # examples using bokecc
+        player_params_str = self._html_search_regex(
+            r'<script[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
+            webpage, 'player params', default=None)
+
+        player_params = compat_parse_qs(player_params_str)
+
+        info_xml = self._download_xml(
+            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
+                player_params['siteid'][0], player_params['vid'][0]), video_id)
+
+        return [{
+            'format_id': 'bokecc',
+            'url': quality.find('./copy').attrib['playurl'],
+            'preference': int(quality.attrib['value']),
+        } for quality in info_xml.findall('./video/quality')]
+
+    def _extract_rtmp_videos(self, webpage):
+        # The server URL is hardcoded
+        video_url = 'rtmpe://video.infoq.com/cfx/st/'
+
+        # Extract video URL
+        encoded_id = self._search_regex(
+            r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id', default=None)
+
+        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
+        playpath = 'mp4:' + real_id
+
+        return [{
+            'format_id': 'rtmp',
+            'url': video_url,
+            'ext': determine_ext(playpath),
+            'play_path': playpath,
+        }]
+
+    def _extract_http_videos(self, webpage):
+        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
+
+        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
+        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
+        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
+
+        return [{
+            'format_id': 'http',
+            'url': http_video_url,
+            'http_headers': {
+                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
+                    policy, signature, key_pair_id),
+            },
+        }]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -30,37 +99,12 @@ class InfoQIE(InfoExtractor):
         video_title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
         video_description = self._html_search_meta('description', webpage, 'description')
 
-        # The server URL is hardcoded
-        video_url = 'rtmpe://video.infoq.com/cfx/st/'
+        if '/cn/' in url:
+            # for China videos, HTTP video URL exists but always fails with 403
+            formats = self._extract_bokecc_videos(webpage, video_id)
+        else:
+            formats = self._extract_rtmp_videos(webpage) + self._extract_http_videos(webpage)
 
-        # Extract video URL
-        encoded_id = self._search_regex(
-            r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id')
-        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
-        playpath = 'mp4:' + real_id
-
-        video_filename = playpath.split('/')[-1]
-        video_id, extension = video_filename.split('.')
-
-        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
-
-        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
-        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
-        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
-
-        formats = [{
-            'format_id': 'rtmp',
-            'url': video_url,
-            'ext': extension,
-            'play_path': playpath,
-        }, {
-            'format_id': 'http',
-            'url': http_video_url,
-            'http_headers': {
-                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
-                    policy, signature, key_pair_id),
-            },
-        }]
         self._sort_formats(formats)
 
         return {

From ee86e2c6d714bf9641075623bf01effac5240fa8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Dec 2015 19:16:01 +0600
Subject: [PATCH 2136/2145] [novamov] Add support for mobile URLs

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 422fb993d..97f0549d3 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -17,7 +17,7 @@ class NovaMovIE(InfoExtractor):
     IE_NAME = 'novamov'
     IE_DESC = 'NovaMov'
 
-    _VALID_URL_TEMPLATE = r'http://(?:(?:www\.)?%(host)s/(?:file|video)/|(?:(?:embed|www)\.)%(host)s/embed\.php\?(?:.*?&)?v=)(?P<id>[a-z\d]{13})'
+    _VALID_URL_TEMPLATE = r'http://(?:(?:www\.)?%(host)s/(?:file|video|mobile/#/videos)/|(?:(?:embed|www)\.)%(host)s/embed\.php\?(?:.*?&)?v=)(?P<id>[a-z\d]{13})'
     _VALID_URL = _VALID_URL_TEMPLATE % {'host': 'novamov\.com'}
 
     _HOST = 'www.novamov.com'

From 2d9295643ef2a94880ab96f485f2d7c261856c6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Dec 2015 23:55:10 +0600
Subject: [PATCH 2137/2145] [footyroom] Skip test

---
 youtube_dl/extractor/footyroom.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 4c7dbca40..370fd006f 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -13,6 +13,7 @@ class FootyRoomIE(InfoExtractor):
             'title': 'Schalke 04 0 – 2 Real Madrid',
         },
         'playlist_count': 3,
+        'skip': 'Video for this match is not available',
     }, {
         'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
         'info_dict': {

From 6583c741cd31b0ebd605efdbde15a81dd6f81de1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 14 Dec 2015 02:33:40 +0800
Subject: [PATCH 2138/2145] [novamov] Fix filekey extraction and reupload test
 video

---
 youtube_dl/extractor/novamov.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 97f0549d3..bbb5be492 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -59,7 +59,7 @@ class NovaMovIE(InfoExtractor):
                 self._FILEKEY_REGEX, webpage, 'filekey', default=default)
             if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
                 return self._search_regex(
-                    r'var\s*%s\s*=\s*"([^"]+)"', webpage, 'filekey', default=default)
+                    r'var\s*%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
             else:
                 return filekey
 
@@ -141,15 +141,14 @@ class NowVideoIE(NovaMovIE):
     _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
 
     _TEST = {
-        'url': 'http://www.nowvideo.to/video/0mw0yow7b6dxa',
-        'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
+        'url': 'http://www.nowvideo.sx/video/f1d6fce9a968b',
+        'md5': '12c82cad4f2084881d8bc60ee29df092',
         'info_dict': {
-            'id': '0mw0yow7b6dxa',
+            'id': 'f1d6fce9a968b',
             'ext': 'flv',
-            'title': 'youtubedl test video _BaW_jenozKc.mp4',
+            'title': 'youtubedl test video BaWjenozKc',
             'description': 'Description',
         },
-        'skip': 'Video 0mw0yow7b6dxa does not exist',
     }
 
 

From eb0bdc2c3e112801bbc7517def25647b8a9fe4a4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 14 Dec 2015 02:50:59 +0800
Subject: [PATCH 2139/2145] [novamov] Fix again

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index bbb5be492..d68c1ad79 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -59,7 +59,7 @@ class NovaMovIE(InfoExtractor):
                 self._FILEKEY_REGEX, webpage, 'filekey', default=default)
             if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
                 return self._search_regex(
-                    r'var\s*%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
+                    r'var\s+%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
             else:
                 return filekey
 

From 31b2051e211f3e2691a186d16733cf91eb4ab391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Dec 2015 21:30:58 +0600
Subject: [PATCH 2140/2145] [utils] Add remove_quotes

---
 test/test_utils.py  | 10 ++++++++++
 youtube_dl/utils.py |  9 +++++++++
 2 files changed, 19 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 501355c74..8fc74e591 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -43,6 +43,7 @@ from youtube_dl.utils import (
     sanitize_path,
     prepend_extension,
     replace_extension,
+    remove_quotes,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -200,6 +201,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
 
+    def test_remove_quotes(self):
+        self.assertEqual(remove_quotes(None), None)
+        self.assertEqual(remove_quotes('"'), '"')
+        self.assertEqual(remove_quotes("'"), "'")
+        self.assertEqual(remove_quotes(';'), ';')
+        self.assertEqual(remove_quotes('";'), '";')
+        self.assertEqual(remove_quotes('""'), '')
+        self.assertEqual(remove_quotes('";"'), ';')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d0606b4bc..91917fc96 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1406,6 +1406,15 @@ def remove_end(s, end):
     return s
 
 
+def remove_quotes(s):
+    if s is None or len(s) < 2:
+        return s
+    for quote in ('"', "'", ):
+        if s[0] == quote and s[-1] == quote:
+            return s[1:-1]
+    return s
+
+
 def url_basename(url):
     path = compat_urlparse.urlparse(url).path
     return path.strip('/').split('/')[-1]

From 0cb58b0259de0b0f44b0326d492b98a8eeb6316e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Dec 2015 21:31:53 +0600
Subject: [PATCH 2141/2145] [youtube] Extract alt_title and creator for music
 videos (Closes #7862)

---
 youtube_dl/extractor/youtube.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9b39505ba..4556a16fb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,6 +33,7 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    remove_quotes,
     remove_start,
     sanitized_Request,
     smuggle_url,
@@ -395,12 +396,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
+                'alt_title': 'I Love It (feat. Charli XCX)',
                 'description': 'md5:782e8651347686cba06e58f71ab51773',
                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
                          'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
+                'creator': 'Icona Pop',
             }
         },
         {
@@ -411,9 +414,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20130703',
                 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
+                'alt_title': 'Tunnel Vision',
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'creator': 'Justin Timberlake',
                 'age_limit': 18,
             }
         },
@@ -492,10 +497,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'nfWlot6h_JM',
                 'ext': 'm4a',
                 'title': 'Taylor Swift - Shake It Off',
+                'alt_title': 'Shake It Off',
                 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
+                'creator': 'Taylor Swift',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -551,9 +558,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20100430',
                 'uploader_id': 'deadmau5',
+                'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
                 'title': 'Deadmau5 - Some Chords (HD)',
+                'alt_title': 'Some Chords',
             },
             'expected_warnings': [
                 'DASH manifest missing',
@@ -701,10 +710,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'lsguqyKfVQg',
                 'ext': 'mp4',
                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
+                'alt_title': 'Dark Walk',
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
                 'uploader': 'IronSoulElf',
+                'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
             },
             'params': {
                 'skip_download': True,
@@ -1308,6 +1319,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
         upload_date = unified_strdate(upload_date)
 
+        m_music = re.search(
+            r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
+            video_webpage)
+        if m_music:
+            video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
+            video_creator = clean_html(m_music.group('creator'))
+        else:
+            video_alt_title = video_creator = None
+
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
             video_webpage, 'categories', default=None)
@@ -1537,7 +1557,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
             'upload_date': upload_date,
+            'creator': video_creator,
             'title': video_title,
+            'alt_title': video_alt_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
             'categories': video_categories,

From d7ffcfcf9703f1f02e642d0855c0977056e2d0fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:09:14 +0600
Subject: [PATCH 2142/2145] [tf1] Fix extraction (Closes #7873)

---
 youtube_dl/extractor/tf1.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 3a68eaa80..32f5f78bf 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -43,11 +43,9 @@ class TF1IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        embed_url = self._html_search_regex(
-            r'["\'](https?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
-        embed_page = self._download_webpage(embed_url, video_id,
-                                            'Downloading embed player page')
-        wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')
+        wat_id = self._html_search_regex(
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d+)\1',
+            webpage, 'wat id', group='id')
         wat_info = self._download_json(
             'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)
         return self.url_result(wat_info['media']['url'], 'Wat')

From ae5e94808e70e608e6bbdf71c0fb8436bcaa76e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:11:52 +0600
Subject: [PATCH 2143/2145] [tf1] Fix extraction (2)

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 32f5f78bf..2d84c9bf8 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -44,7 +44,7 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d+)\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
             webpage, 'wat id', group='id')
         wat_info = self._download_json(
             'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)

From 05467d5a5211adb5e2473decaef82769e778ffc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:31:58 +0600
Subject: [PATCH 2144/2145] [tf1] Relax _VALID_URL

---
 youtube_dl/extractor/tf1.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 2d84c9bf8..80cd92938 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
@@ -38,6 +38,9 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.tf1.fr/hd1/documentaire/videos/mylene-farmer-d-une-icone.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ad1b6017cd48c62896a2d6c978f21e27f5d96d2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:36:59 +0600
Subject: [PATCH 2145/2145] [tf1] Fix tests

---
 youtube_dl/extractor/tf1.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 80cd92938..6890021cf 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -22,7 +22,7 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
         'info_dict': {
-            'id': '12043945',
+            'id': 'le-grand-mysterioso-chuggington-7085291-739',
             'ext': 'mp4',
             'title': 'Le grand Mystérioso - Chuggington',
             'description': 'Le grand Mystérioso - Emery rêve qu\'un article lui soit consacré dans le journal.',
@@ -32,6 +32,7 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
+        'skip': 'HTTP Error 410: Gone',
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
         'only_matching': True,