From 63fc80005782430180ed0d13ac6ab5ca497d333a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Mar 2015 23:17:19 +0800
Subject: [PATCH 0001/1515] [Letv] Fix test_Letv and test_Letv_1 failures in
 python 3

---
 youtube_dl/extractor/letv.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 85eee141b..9ed81a199 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -88,9 +88,10 @@ class LetvIE(InfoExtractor):
         play_json_req = compat_urllib_request.Request(
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
-        play_json_req.add_header(
-            'Ytdl-request-proxy',
-            self._downloader.params.get('cn_verification_proxy'))
+        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+        if cn_verification_proxy:
+            play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
         play_json = self._download_json(
             play_json_req,
             media_id, 'playJson data')

From 5ee6fc974e617ce2f8d9d62c416091a1daa6d802 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 02:43:05 +0800
Subject: [PATCH 0002/1515] [sohu] Fix info extractor and add tests

---
 youtube_dl/extractor/sohu.py | 46 +++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index c04791997..ef7ec51df 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -4,22 +4,42 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .common import compat_str
+from ..utils import compat_str
+from ..compat import compat_urllib_request
 
 
 class SohuIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
 
-    _TEST = {
+    _TESTS = [{
+        'note': 'This video is available only in Mainland China',
         'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
-        'md5': 'bde8d9a6ffd82c63a1eefaef4eeefec7',
+        'md5': '29175c8cadd8b5cc4055001e85d6b372',
         'info_dict': {
             'id': '382479172',
             'ext': 'mp4',
             'title': 'MV：Far East Movement《The Illest》',
         },
-        'skip': 'Only available from China',
-    }
+        'params': {
+            'cn_verification_proxy': 'proxy.uku.im:8888'
+        }
+    }, {
+        'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
+        'md5': '699060e75cf58858dd47fb9c03c42cfb',
+        'info_dict': {
+            'id': '409385080',
+            'ext': 'mp4',
+            'title': '《2015湖南卫视羊年元宵晚会》唐嫣《花好月圆》',
+        }
+    }, {
+        'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
+        'md5': '9bf34be48f2f4dadcb226c74127e203c',
+        'info_dict': {
+            'id': '78693464',
+            'ext': 'mp4',
+            'title': '【爱范品】第31期：MWC见不到的奇葩手机',
+        }
+    }]
 
     def _real_extract(self, url):
 
@@ -29,9 +49,14 @@ class SohuIE(InfoExtractor):
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
 
-            return self._download_json(
-                base_data_url + vid_id, video_id,
-                'Downloading JSON data for %s' % vid_id)
+            req = compat_urllib_request.Request(base_data_url + vid_id)
+
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
+            return self._download_json(req, video_id,
+                                       'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -77,6 +102,11 @@ class SohuIE(InfoExtractor):
                     % (format_id, i + 1, part_count))
 
                 part_info = part_str.split('|')
+
+                # Sanitize URL to prevent download failure
+                if part_info[0][-1] == '/' and su[i][0] == '/':
+                    su[i] = su[i][1:]
+
                 video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
 
                 formats.append({

From 5c7495a19429e3b27c003a4bd5bb96ed1e3a4932 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 02:48:27 +0800
Subject: [PATCH 0003/1515] [sohu] Correct wrong imports

---
 youtube_dl/extractor/sohu.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ef7ec51df..335e84fab 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_str
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_str,
+    compat_urllib_request
+)
 
 
 class SohuIE(InfoExtractor):

From 55969016e96fded28b97b2ef3bbf66efa83d6afb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 12:43:49 +0800
Subject: [PATCH 0004/1515] [utils] Add a function to sanitize consecutive
 slashes in URLs

---
 test/test_utils.py           | 16 ++++++++++++++++
 youtube_dl/extractor/sohu.py |  8 +++-----
 youtube_dl/utils.py          | 15 +++++++++++++++
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 64fad58ad..e02069c4d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -54,6 +54,7 @@ from youtube_dl.utils import (
     xpath_with_ns,
     render_table,
     match_str,
+    url_sanitize_consecutive_slashes,
 )
 
 
@@ -501,6 +502,21 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
+    def test_url_sanitize_consecutive_slashes(self):
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/foo//bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname//foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname//'), 'http://hostname/')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/'), 'http://hostname/')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 335e84fab..5adc734d9 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,6 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
+from ..utils import url_sanitize_consecutive_slashes
 
 
 class SohuIE(InfoExtractor):
@@ -105,11 +106,8 @@ class SohuIE(InfoExtractor):
 
                 part_info = part_str.split('|')
 
-                # Sanitize URL to prevent download failure
-                if part_info[0][-1] == '/' and su[i][0] == '/':
-                    su[i] = su[i][1:]
-
-                video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+                video_url = url_sanitize_consecutive_slashes(
+                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({
                     'url': video_url,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7426e2a1f..ef14f9a36 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1789,3 +1789,18 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
+
+
+def url_sanitize_consecutive_slashes(url):
+    """Sanitize URLs with consecutive slashes
+
+    For example, transform both
+        http://hostname/foo//bar/filename.html
+    and
+        http://hostname//foo/bar/filename.html
+    into
+        http://hostname/foo/bar/filename.html
+    """
+    parsed_url = list(compat_urlparse.urlparse(url))
+    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
+    return compat_urlparse.urlunparse(parsed_url)

From a172d96292abda037b23b1f2b14dd2c6e65b56bb Mon Sep 17 00:00:00 2001
From: bonfy <foreverbonfy@163.com>
Date: Sat, 7 Mar 2015 14:05:56 +0800
Subject: [PATCH 0005/1515] [douyutv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/douyutv.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/douyutv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..7d0798176 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
+from .douyutv import DouyutvIE
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
new file mode 100644
index 000000000..e9b92eb3b
--- /dev/null
+++ b/youtube_dl/extractor/douyutv.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+)
+
+class DouyutvIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
+
+    '''
+    show_status: 1 直播中 ，2 没有直播
+    '''
+
+    _TEST = {
+        'url': 'http://www.douyutv.com/iseven',
+        'info_dict': {
+            'id': 'iseven',
+            'title': '清晨醒脑！T-ara根本停不下来！',
+            'ext': 'flv',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'is_live': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info_url = 'http://www.douyutv.com/api/client/room/' + video_id
+
+        config = self._download_json(info_url, video_id)
+
+        error_code = config.get('error')
+        show_status = config['data'].get('show_status')
+        if error_code is not 0:
+            raise ExtractorError('Server reported error %i' % error_code,
+                                 expected=True)
+
+        if show_status == '2':
+            raise ExtractorError('The live show has not yet started',
+                                 expected=True)
+
+        title = config['data'].get('room_name')
+        rtmp_url = config['data'].get('rtmp_url')
+        rtmp_live = config['data'].get('rtmp_live')
+        thumbnail = config['data'].get('room_src') 
+
+        url = rtmp_url+'/'+rtmp_live
+
+        return {
+            'id': video_id,
+            'title': title,
+            'ext':'flv',
+            'url': url,
+            'thumbnail': thumbnail,
+            'is_live': True,
+            # TODO more properties (see youtube_dl/extractor/common.py)
+        }
\ No newline at end of file

From cd65491c306f644d7bb3c7ad98795a3f8660be49 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Mar 2015 00:59:49 +0800
Subject: [PATCH 0006/1515] [Sohu] Add a multiplart video test case

---
 youtube_dl/extractor/sohu.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 5adc734d9..7db5b2f13 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -42,6 +42,37 @@ class SohuIE(InfoExtractor):
             'ext': 'mp4',
             'title': '【爱范品】第31期：MWC见不到的奇葩手机',
         }
+    }, {
+        'note': 'Multipart video',
+        'url': 'http://my.tv.sohu.com/pl/8384802/78910339.shtml',
+        'info_dict': {
+            'id': '78910339',
+        },
+        'playlist': [{
+            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
+            'info_dict': {
+                'id': '78910339_part1',
+                'ext': 'mp4',
+                'duration': 294,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }, {
+            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
+            'info_dict': {
+                'id': '78910339_part2',
+                'ext': 'mp4',
+                'duration': 300,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }, {
+            'md5': '8407e634175fdac706766481b9443450',
+            'info_dict': {
+                'id': '78910339_part3',
+                'ext': 'mp4',
+                'duration': 150,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }]
     }]
 
     def _real_extract(self, url):

From 2cb434e53ee861c8bcbd538455be107085f444ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Mar 2015 01:05:01 +0800
Subject: [PATCH 0007/1515] [Sohu] Fix title extraction

---
 youtube_dl/extractor/sohu.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7db5b2f13..ea5cc06b9 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -73,6 +73,17 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }]
+    }, {
+        'info': 'Video with title containing dash',
+        'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
+        'info_dict': {
+            'id': '78932792',
+            'ext': 'mp4',
+            'title': 'youtube-dl testing video',
+        },
+        'params': {
+            'skip_download': True
+        }
     }]
 
     def _real_extract(self, url):
@@ -97,10 +108,8 @@ class SohuIE(InfoExtractor):
         mytv = mobj.group('mytv') is not None
 
         webpage = self._download_webpage(url, video_id)
-        raw_title = self._html_search_regex(
-            r'(?s)<title>(.+?)</title>',
-            webpage, 'video title')
-        title = raw_title.partition('-')[0].strip()
+
+        title = self._og_search_title(webpage)
 
         vid = self._html_search_regex(
             r'var vid ?= ?["\'](\d+)["\']',

From 2e90dff2c2ecade8afb444b086fbc0ad6d2c812d Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 16 Mar 2015 20:05:02 +0100
Subject: [PATCH 0008/1515] The Daily Show Podcast support

---
 youtube_dl/extractor/__init__.py      |  3 +-
 youtube_dl/extractor/comedycentral.py | 21 ++++++++++++++
 youtube_dl/extractor/libsyn.py        | 41 +++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/libsyn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1bb3e1a1c..e94779d40 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -84,7 +84,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE, TheDailyShowPodcastIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
@@ -250,6 +250,7 @@ from .letv import (
     LetvPlaylistIE
 )
 from .lifenews import LifeNewsIE
+from .libsyn import LibsynIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e5edcc84b..e427b9821 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
+from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,
@@ -272,3 +273,23 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
+
+class TheDailyShowPodcastIE(InfoExtractor):
+    _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        player_url = self._search_regex(r'<iframe(?:\s+[^>]+)?\s*src="((?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/[0-9]+)', webpage, 'player URL')
+        if player_url.startswith('//'):
+            mobj = re.match(self._VALID_URL, url)
+            scheme = mobj.group('scheme')
+            if not scheme:
+                scheme = 'https:'
+            player_url = scheme + player_url
+
+        return {
+            '_type': 'url_transparent',
+            'url': player_url,
+        }
diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
new file mode 100644
index 000000000..4b5029f89
--- /dev/null
+++ b/youtube_dl/extractor/libsyn.py
@@ -0,0 +1,41 @@
+# encoding: utf-8
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+class LibsynIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
+
+    def _real_extract(self, url):
+        if url.startswith('//'):
+            url = 'https:' + url
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        podcast_title         = self._search_regex(r'<h2>(.*?)</h2>', webpage, 'show title')
+        podcast_episode_title = self._search_regex(r'<h3>(.*?)</h3>', webpage, 'episode title')
+        podcast_date          = unified_strdate(self._search_regex(r'<div class="release_date">Released: (.*?)</div>', webpage, 'release date'))
+        podcast_description   = self._search_regex(r'<div id="info_text_body">(.*?)</div>', webpage, 'description')
+
+        url0 = self._search_regex(r'var mediaURLLibsyn = "(?P<url0>https?://.*)";', webpage, 'first media URL')
+        url1 = self._search_regex(r'var mediaURL = "(?P<url1>https?://.*)";', webpage, 'second media URL')
+
+        if url0 != url1:
+            formats = [{
+                'url': url0
+            }, {
+                'url': url1
+            }]
+        else:
+            formats = [{
+                'url': url0
+            }]
+
+        return {
+            'id': display_id,
+            'title': podcast_episode_title,
+            'description': podcast_description,
+            'upload_date': podcast_date,
+            'formats': formats,
+        }

From c06a9f8730f41fb24835d482114403f227bc8421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 19:42:50 +0600
Subject: [PATCH 0009/1515] [arte+7] Check formats (Closes #5224)

---
 youtube_dl/extractor/arte.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 929dd3cc5..8273bd6c9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -146,6 +146,7 @@ class ArteTVPlus7IE(InfoExtractor):
 
             formats.append(format)
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         info_dict['formats'] = formats

From 576904bce64ca036a7b21b43fa3f8c023e0bcdb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 20:01:31 +0600
Subject: [PATCH 0010/1515] [letv] Clarify download message

---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9ed81a199..1484ac0d2 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -94,7 +94,7 @@ class LetvIE(InfoExtractor):
 
         play_json = self._download_json(
             play_json_req,
-            media_id, 'playJson data')
+            media_id, 'Downloading playJson data')
 
         # Check for errors
         playstatus = play_json['playstatus']

From 733be371af58ec63be5faa52ee24cab4dd85d388 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Thu, 15 Jan 2015 21:28:57 -0500
Subject: [PATCH 0011/1515] Add megavideoz.eu support.

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/megavideozeu.py | 39 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1bb3e1a1c..5316af2d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -267,6 +267,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
+from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
new file mode 100644
index 000000000..e77b5f734
--- /dev/null
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+)
+
+
+class MegavideozeuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        tmp_video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, tmp_video_id)
+
+        config_php = self._html_search_regex(
+            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+
+	configpage = self._download_webpage(config_php, tmp_video_id)
+
+        video_id = self._html_search_regex(
+            r'<mediaid>([^<]+)', configpage, 'video id')
+        video_url = self._html_search_regex(
+            r'<file>([^<]+)', configpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
+        duration = int_or_none(self._html_search_regex(
+            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'duration': duration
+        }

From d41a3fa1b47fc050e206bd16feed02b6a62e7d2d Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 15:02:13 -0500
Subject: [PATCH 0012/1515] [Primesharetv] Add primeshare.tv extractor, still
 need test data

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/primesharetv.py | 46 ++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/primesharetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5316af2d1..b02812365 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -381,6 +381,7 @@ from .pornhub import (
 )
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
+from .primesharetv import PrimesharetvIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
new file mode 100644
index 000000000..967125abc
--- /dev/null
+++ b/youtube_dl/extractor/primesharetv.py
@@ -0,0 +1,46 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+    urlencode_postdata,
+)
+from ..compat import (
+    compat_urllib_request,
+)
+
+class PrimesharetvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+       
+        self._sleep(9, video_id)
+        
+        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
+        data = urlencode_postdata({
+            'hash': hashtoken,
+        })
+        headers = {
+            'Referer': url,
+            'Content-Type': 'application/x-www-form-urlencoded',
+        }
+        video_page_request = compat_urllib_request.Request(url, data, headers=headers)
+        video_page = self._download_webpage(video_page_request, None, False, '')
+
+        video_url = self._html_search_regex(
+            r'url: \'(http://l\.primeshare\.tv[^\']+)\',', video_page, 'video url')
+
+        title = self._html_search_regex(
+            r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': 'mp4',
+        }

From af69cab21d48da78ff7d2b54effb9dfd5fcfd1d8 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 15:18:06 -0500
Subject: [PATCH 0013/1515] [Primesharetv] Add public domain example video

---
 youtube_dl/extractor/primesharetv.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 967125abc..7c545761b 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -15,6 +15,19 @@ from ..compat import (
 class PrimesharetvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
 
+    _TESTS = [
+        {
+            'url': 'http://primeshare.tv/download/238790B611',
+            'md5': 'bb41f9f6c0dd434c729f04ce5b677192',
+            'info_dict': {
+                'id': '238790B611',
+                'ext': 'mp4',
+                "title": "Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona [...]",
+                "duration": 10,
+            },
+        }
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -33,7 +46,7 @@ class PrimesharetvIE(InfoExtractor):
         video_page = self._download_webpage(video_page_request, None, False, '')
 
         video_url = self._html_search_regex(
-            r'url: \'(http://l\.primeshare\.tv[^\']+)\',', video_page, 'video url')
+            r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
 
         title = self._html_search_regex(
             r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')

From 13047f41359e32bf56bdc8dd555bae6c1065512d Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 17:38:21 -0500
Subject: [PATCH 0014/1515] [Primesharetv] Handle file not existing properly.

---
 youtube_dl/extractor/primesharetv.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 7c545761b..570fd2210 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,8 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     int_or_none,
     parse_filesize,
     unified_strdate,
@@ -31,10 +34,12 @@ class PrimesharetvIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        if re.search(r'<h1>File not exist</h1>', webpage) is not None:
+            raise ExtractorError('The file does not exist', expected=True)
+        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
        
         self._sleep(9, video_id)
         
-        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
         data = urlencode_postdata({
             'hash': hashtoken,
         })
@@ -44,7 +49,6 @@ class PrimesharetvIE(InfoExtractor):
         }
         video_page_request = compat_urllib_request.Request(url, data, headers=headers)
         video_page = self._download_webpage(video_page_request, None, False, '')
-
         video_url = self._html_search_regex(
             r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
 
@@ -57,3 +61,8 @@ class PrimesharetvIE(InfoExtractor):
             'title': title,
             'ext': 'mp4',
         }
+    
+    def _debug_print(self, txt):
+        if self._downloader.params.get('verbose'):
+            self.to_screen('[debug] %s' % txt)
+

From 0499cd866e7e746658f33e6c2f44f7e1e699ad1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:06:38 +0600
Subject: [PATCH 0015/1515] [primesharetv] Clean up

---
 youtube_dl/extractor/__init__.py     |  2 +-
 youtube_dl/extractor/primesharetv.py | 83 ++++++++++++++--------------
 2 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b02812365..3c5401145 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -381,7 +381,7 @@ from .pornhub import (
 )
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
-from .primesharetv import PrimesharetvIE
+from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 570fd2210..01cc3d9ea 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,59 +1,65 @@
-# encoding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-    urlencode_postdata,
-)
 from ..compat import (
+    compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import ExtractorError
 
-class PrimesharetvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
 
-    _TESTS = [
-        {
-            'url': 'http://primeshare.tv/download/238790B611',
-            'md5': 'bb41f9f6c0dd434c729f04ce5b677192',
-            'info_dict': {
-                'id': '238790B611',
-                'ext': 'mp4',
-                "title": "Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona [...]",
-                "duration": 10,
-            },
-        }
-    ]
+class PrimeShareTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>[\da-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://primeshare.tv/download/238790B611',
+        'md5': 'b92d9bf5461137c36228009f31533fbc',
+        'info_dict': {
+            'id': '238790B611',
+            'ext': 'mp4',
+            'title': 'Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona',
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        if re.search(r'<h1>File not exist</h1>', webpage) is not None:
-            raise ExtractorError('The file does not exist', expected=True)
-        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
-       
-        self._sleep(9, video_id)
-        
-        data = urlencode_postdata({
-            'hash': hashtoken,
-        })
+
+        if '>File not exist<' in webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            (?:id="[^"]+"\s+)?
+            value="([^"]*)"
+            ''', webpage))
+
         headers = {
             'Referer': url,
             'Content-Type': 'application/x-www-form-urlencoded',
         }
-        video_page_request = compat_urllib_request.Request(url, data, headers=headers)
-        video_page = self._download_webpage(video_page_request, None, False, '')
-        video_url = self._html_search_regex(
-            r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
+
+        wait_time = int(self._search_regex(
+            r'var\s+cWaitTime\s*=\s*(\d+)',
+            webpage, 'wait time', default=7)) + 1
+        self._sleep(wait_time, video_id)
+
+        req = compat_urllib_request.Request(
+            url, compat_urllib_parse.urlencode(fields), headers)
+        video_page = self._download_webpage(
+            req, video_id, 'Downloading video page')
+
+        video_url = self._search_regex(
+            r"url\s*:\s*'([^']+\.primeshare\.tv(?::443)?/file/[^']+)'",
+            video_page, 'video url')
 
         title = self._html_search_regex(
-            r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')
+            r'<h1>Watch\s*(?:&nbsp;)?\s*\((.+?)(?:\s*\[\.\.\.\])?\)\s*(?:&nbsp;)?\s*<strong>',
+            video_page, 'title')
 
         return {
             'id': video_id,
@@ -61,8 +67,3 @@ class PrimesharetvIE(InfoExtractor):
             'title': title,
             'ext': 'mp4',
         }
-    
-    def _debug_print(self, txt):
-        if self._downloader.params.get('verbose'):
-            self.to_screen('[debug] %s' % txt)
-

From 219da6bb685765186b7ffb878399c32f44351802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:13:42 +0600
Subject: [PATCH 0016/1515] [megavideoeu] Remove extractor

---
 youtube_dl/extractor/__init__.py     |  1 -
 youtube_dl/extractor/megavideozeu.py | 39 ----------------------------
 2 files changed, 40 deletions(-)
 delete mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3c5401145..bceed92e1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -267,7 +267,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
deleted file mode 100644
index e77b5f734..000000000
--- a/youtube_dl/extractor/megavideozeu.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-)
-
-
-class MegavideozeuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
-
-    def _real_extract(self, url):
-        tmp_video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, tmp_video_id)
-
-        config_php = self._html_search_regex(
-            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
-
-	configpage = self._download_webpage(config_php, tmp_video_id)
-
-        video_id = self._html_search_regex(
-            r'<mediaid>([^<]+)', configpage, 'video id')
-        video_url = self._html_search_regex(
-            r'<file>([^<]+)', configpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
-        duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'duration': duration
-        }

From 92a4793b3cac36677bed5423b12b3d97d15c6036 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:34:22 +0600
Subject: [PATCH 0017/1515] [utils] Place sanitize url function near other
 sanitizing functions

---
 test/test_utils.py  | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py | 22 +++++++---------------
 2 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4f0ffd482..3431ad24e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -39,6 +39,7 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
+    sanitize_url_path_consecutive_slashes,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -55,7 +56,6 @@ from youtube_dl.utils import (
     xpath_with_ns,
     render_table,
     match_str,
-    url_sanitize_consecutive_slashes,
 )
 
 
@@ -169,6 +169,26 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
+    def test_sanitize_url_path_consecutive_slashes(self):
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/foo//bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname//foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname//'),
+            'http://hostname/')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/'),
+            'http://hostname/')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
+            'http://hostname/abc/')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
@@ -539,21 +559,6 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
-    def test_url_sanitize_consecutive_slashes(self):
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/foo//bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname//foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname//'), 'http://hostname/')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/'), 'http://hostname/')
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e82e3998a..472d4df41 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -326,6 +326,13 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
+def sanitize_url_path_consecutive_slashes(url):
+    """Collapses consecutive slashes in URLs' path"""
+    parsed_url = list(compat_urlparse.urlparse(url))
+    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
+    return compat_urlparse.urlunparse(parsed_url)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []
@@ -1804,18 +1811,3 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
-
-
-def url_sanitize_consecutive_slashes(url):
-    """Sanitize URLs with consecutive slashes
-
-    For example, transform both
-        http://hostname/foo//bar/filename.html
-    and
-        http://hostname//foo/bar/filename.html
-    into
-        http://hostname/foo/bar/filename.html
-    """
-    parsed_url = list(compat_urlparse.urlparse(url))
-    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
-    return compat_urlparse.urlunparse(parsed_url)

From cd459b1d490ca8c0639220a835f5e6bee3e9a80d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:39:31 +0600
Subject: [PATCH 0018/1515] [sohu] Fix test's note info

---
 youtube_dl/extractor/sohu.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ea5cc06b9..11edf616a 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import url_sanitize_consecutive_slashes
+from ..utils import sanitize_url_path_consecutive_slashes
 
 
 class SohuIE(InfoExtractor):
@@ -74,7 +74,7 @@ class SohuIE(InfoExtractor):
             }
         }]
     }, {
-        'info': 'Video with title containing dash',
+        'note': 'Video with title containing dash',
         'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
         'info_dict': {
             'id': '78932792',
@@ -100,8 +100,9 @@ class SohuIE(InfoExtractor):
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
-            return self._download_json(req, video_id,
-                                       'Downloading JSON data for %s' % vid_id)
+            return self._download_json(
+                req, video_id,
+                'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -146,7 +147,7 @@ class SohuIE(InfoExtractor):
 
                 part_info = part_str.split('|')
 
-                video_url = url_sanitize_consecutive_slashes(
+                video_url = sanitize_url_path_consecutive_slashes(
                     '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({

From 2ca1c5aa9f8d7db08608de0bd99f6c994ab496d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 22:27:33 +0600
Subject: [PATCH 0019/1515] [douyutv] Improve and extract all formats

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/douyutv.py  | 72 ++++++++++++++++++++------------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3901bfc3..73c17aa84 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -107,7 +107,7 @@ from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
-from .douyutv import DouyutvIE
+from .douyutv import DouyuTVIE
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index e9b92eb3b..d7956e6e4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -2,58 +2,76 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
-from ..utils import (
-    ExtractorError,
-)
 
-class DouyutvIE(InfoExtractor):
+class DouyuTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
-
-    '''
-    show_status: 1 直播中 ，2 没有直播
-    '''
-
     _TEST = {
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
             'id': 'iseven',
-            'title': '清晨醒脑！T-ara根本停不下来！',
             'ext': 'flv',
+            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:9e525642c25a0a24302869937cf69d17',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': '7师傅',
+            'uploader_id': '431925',
             'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        info_url = 'http://www.douyutv.com/api/client/room/' + video_id
 
-        config = self._download_json(info_url, video_id)
+        config = self._download_json(
+            'http://www.douyutv.com/api/client/room/%s' % video_id, video_id)
 
-        error_code = config.get('error')
-        show_status = config['data'].get('show_status')
+        data = config['data']
+
+        error_code = config.get('error', 0)
+        show_status = data.get('show_status')
         if error_code is not 0:
-            raise ExtractorError('Server reported error %i' % error_code,
-                                 expected=True)
+            raise ExtractorError(
+                'Server reported error %i' % error_code, expected=True)
 
+        # 1 = live, 2 = offline
         if show_status == '2':
-            raise ExtractorError('The live show has not yet started',
-                                 expected=True)
+            raise ExtractorError(
+                'Live stream is offline', expected=True)
 
-        title = config['data'].get('room_name')
-        rtmp_url = config['data'].get('rtmp_url')
-        rtmp_live = config['data'].get('rtmp_live')
-        thumbnail = config['data'].get('room_src') 
+        base_url = data['rtmp_url']
+        live_path = data['rtmp_live']
 
-        url = rtmp_url+'/'+rtmp_live
+        title = self._live_title(data['room_name'])
+        description = data.get('show_details')
+        thumbnail = data.get('room_src')
+
+        uploader = data.get('nickname')
+        uploader_id = data.get('owner_uid')
+
+        multi_formats = data.get('rtmp_multi_bitrate')
+        if not isinstance(multi_formats, dict):
+            multi_formats = {}
+        multi_formats['live'] = live_path
+
+        formats = [{
+            'url': '%s/%s' % (base_url, format_path),
+            'format_id': format_id,
+            'preference': 1 if format_id == 'live' else 0,
+        } for format_id, format_path in multi_formats.items()]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'ext':'flv',
-            'url': url,
+            'description': description,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
             'is_live': True,
-            # TODO more properties (see youtube_dl/extractor/common.py)
-        }
\ No newline at end of file
+        }

From 98f02fdde25bf00fbccfb211abb574f80819a177 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 22:33:05 +0600
Subject: [PATCH 0020/1515] Credit @jbuchbinder for primesharetv (#5123)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 872da6071..512469f4c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -116,3 +116,4 @@ Duncan Keall
 Alexander Mamay
 Devin J. Pohly
 Eduardo Ferro Aldama
+Jeff Buchbinder

From b1337948ebeeacfcf104d513fac5913d3a5818a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 23:13:43 +0600
Subject: [PATCH 0021/1515] [grooveshark] Fix extraction

---
 youtube_dl/extractor/grooveshark.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index 848d17beb..36ad4915c 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -140,9 +140,9 @@ class GroovesharkIE(InfoExtractor):
 
         if webpage is not None:
             o = GroovesharkHtmlParser.extract_object_tags(webpage)
-            return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
+            return webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed']
 
-        return (webpage, None)
+        return webpage, None
 
     def _real_initialize(self):
         self.ts = int(time.time() * 1000)  # timestamp in millis
@@ -154,7 +154,7 @@ class GroovesharkIE(InfoExtractor):
         swf_referer = None
         if self.do_playerpage_request:
             (_, player_objs) = self._get_playerpage(url)
-            if player_objs is not None:
+            if player_objs:
                 swf_referer = self._build_swf_referer(url, player_objs[0])
                 self.to_screen('SWF Referer: %s' % swf_referer)
 

From 84f810160667f9129e6a8d841fae2d0e2e1fec86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 23:51:40 +0600
Subject: [PATCH 0022/1515] [generic] Follow redirects specified by `Refresh`
 HTTP header

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4e6927b08..0b8d96c27 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1270,8 +1270,14 @@ class GenericIE(InfoExtractor):
         if not found:
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
-                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'?([^\'"]+)',
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)',
                 webpage)
+            if not found:
+                # Look also in Refresh HTTP header
+                refresh_header = head_response.headers.get('Refresh')
+                if refresh_header:
+                    found = re.search(
+                        r'[0-9]{,2};\s*(?:URL|url)=(.+)', refresh_header)
             if found:
                 new_url = found.group(1)
                 self.report_following_redirect(new_url)

From 9ef4f12b534578ae3d3e47815492c90826c03c36 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Tue, 17 Mar 2015 18:54:36 +0100
Subject: [PATCH 0023/1515] testcases for libsyn and The Daily Show Podcast
 extractors

---
 youtube_dl/extractor/comedycentral.py | 4 ++++
 youtube_dl/extractor/libsyn.py        | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e427b9821..bd3817b56 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -276,6 +276,10 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
 class TheDailyShowPodcastIE(InfoExtractor):
     _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
+    _TESTS = [{
+        "url": "http://thedailyshow.cc.com/podcast/episodetwelve",
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 4b5029f89..88379f276 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -6,6 +6,15 @@ from ..utils import (
 
 class LibsynIE(InfoExtractor):
     _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
+    _TESTS = [{
+        'url': "http://html5-player.libsyn.com/embed/episode/id/3377616/",
+        'info_dict': {
+            'id': "3377616",
+            'ext': "mp3",
+            'title': "Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+            'description': "<p>Bassem Youssef joins executive producer Steve Bodow and senior producer Sara Taksler for a conversation about how&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">The Daily Show</em>&nbsp;inspired Bassem to create&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Al-Bernameg</em>, his massively popular (and now banned) Egyptian news satire program. Sara discusses her soon-to-be-released documentary,&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Tickling Giants</em>, which chronicles how Bassem and his staff risked their safety every day to tell jokes.</p>",
+        },
+    }]
 
     def _real_extract(self, url):
         if url.startswith('//'):

From ed9a25dd612fb06d9cf007a6491ac9982535a8f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 00:05:40 +0600
Subject: [PATCH 0024/1515] [generic] Generalize redirect regex

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0b8d96c27..dc5755d12 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1268,16 +1268,16 @@ class GenericIE(InfoExtractor):
             # HTML5 video
             found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
+            REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
-                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)',
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
                 webpage)
             if not found:
                 # Look also in Refresh HTTP header
                 refresh_header = head_response.headers.get('Refresh')
                 if refresh_header:
-                    found = re.search(
-                        r'[0-9]{,2};\s*(?:URL|url)=(.+)', refresh_header)
+                    found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
                 new_url = found.group(1)
                 self.report_following_redirect(new_url)

From c3c5c31517ce121740666440a71becaba11176ac Mon Sep 17 00:00:00 2001
From: Todoroki <todoroki.auone@gmail.com>
Date: Wed, 18 Mar 2015 22:19:55 +0900
Subject: [PATCH 0025/1515] fix nm video DL issue when logged in

---
 youtube_dl/extractor/niconico.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 7fb4e57df..a48df7f62 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -89,7 +89,7 @@ class NiconicoIE(InfoExtractor):
         if self._AUTHENTICATED:
             # Get flv info
             flv_info_webpage = self._download_webpage(
-                'http://flapi.nicovideo.jp/api/getflv?v=' + video_id,
+                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
                 video_id, 'Downloading flv info')
         else:
             # Get external player info
@@ -97,7 +97,6 @@ class NiconicoIE(InfoExtractor):
                 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
             thumb_play_key = self._search_regex(
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
-
             # Get flv info
             flv_info_data = compat_urllib_parse.urlencode({
                 'k': thumb_play_key,

From ee580538fa741fc35faa8210e0245e252bf23e40 Mon Sep 17 00:00:00 2001
From: Todoroki <todoroki.auone@gmail.com>
Date: Wed, 18 Mar 2015 22:24:17 +0900
Subject: [PATCH 0026/1515] fix nm video DL issue when logged in

---
 youtube_dl/extractor/niconico.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a48df7f62..4d57ac703 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -97,6 +97,7 @@ class NiconicoIE(InfoExtractor):
                 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
             thumb_play_key = self._search_regex(
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
+
             # Get flv info
             flv_info_data = compat_urllib_parse.urlencode({
                 'k': thumb_play_key,

From 2c2c06e359dc3843a86585a9a4b6419f03af510e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 20:28:00 +0600
Subject: [PATCH 0027/1515] [krasview] Fix extraction (Closes #5228)

---
 youtube_dl/extractor/krasview.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index e46954b47..96f95979a 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -40,8 +40,10 @@ class KrasViewIE(InfoExtractor):
         description = self._og_search_description(webpage, default=None)
         thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
         duration = int_or_none(flashvars.get('duration'))
-        width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
-        height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, 'video width', default=None))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, 'video height', default=None))
 
         return {
             'id': video_id,

From fa8ce2690447fa06a10398efd5b6dfa96c2ed1b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Sun, 15 Mar 2015 22:32:06 +0100
Subject: [PATCH 0028/1515] [mixcloud] Fix extraction like-count

---
 youtube_dl/extractor/mixcloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 1831c6749..7c6c885f9 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -99,8 +99,7 @@ class MixcloudIE(InfoExtractor):
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
         like_count = str_to_int(self._search_regex(
-            [r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
-             r'/favorites/?">([0-9]+)<'],
+            r'\bbutton-favorite\b.+m-ajax-toggle-count="([^"]+)"',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From 7a757b7194b1f1151857503baed3fa51d57397aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Mar 2015 16:50:23 +0100
Subject: [PATCH 0029/1515] [mixcloud] Fix extraction of some metadata

The second test had some wrong info.
I couldn't find the timestamp, so I have removed it.
---
 youtube_dl/extractor/mixcloud.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 7c6c885f9..2fe55179a 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -10,7 +10,6 @@ from ..utils import (
     ExtractorError,
     HEADRequest,
     str_to_int,
-    parse_iso8601,
 )
 
 
@@ -27,8 +26,6 @@ class MixcloudIE(InfoExtractor):
             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
             'uploader': 'Daniel Holbach',
             'uploader_id': 'dholbach',
-            'upload_date': '20111115',
-            'timestamp': 1321359578,
             'thumbnail': 're:https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
@@ -37,12 +34,12 @@ class MixcloudIE(InfoExtractor):
         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
         'info_dict': {
             'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
-            'ext': 'm4a',
-            'title': 'Electric Relaxation vol. 3',
+            'ext': 'mp3',
+            'title': 'Caribou 7 inch Vinyl Mix & Chat',
             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
-            'uploader': 'Daniel Drumz',
+            'uploader': 'Gilles Peterson Worldwide',
             'uploader_id': 'gillespeterson',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': 're:https?://.*/images/',
             'view_count': int,
             'like_count': int,
         },
@@ -85,7 +82,7 @@ class MixcloudIE(InfoExtractor):
             raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
-            r'<span class="play-button[^"]*?"'
+            r'm-play-on-spacebar[^>]+'
             r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
         title = self._html_search_regex(
             PREFIX + r'm-title="([^"]+)"', webpage, 'title')
@@ -105,9 +102,6 @@ class MixcloudIE(InfoExtractor):
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
              r'/listeners/?">([0-9,.]+)</a>'],
             webpage, 'play count', fatal=False))
-        timestamp = parse_iso8601(self._search_regex(
-            r'<time itemprop="dateCreated" datetime="([^"]+)">',
-            webpage, 'upload date', default=None))
 
         return {
             'id': track_id,
@@ -117,7 +111,6 @@ class MixcloudIE(InfoExtractor):
             'thumbnail': thumbnail,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'timestamp': timestamp,
             'view_count': view_count,
             'like_count': like_count,
         }

From aae53774f247edcca7671434c16f7931d5c7b9da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Mon, 16 Mar 2015 00:20:06 +0100
Subject: [PATCH 0030/1515] [mixcloud] Try preview server first, then further
 numbers

---
 youtube_dl/extractor/mixcloud.py | 49 ++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 2fe55179a..21aea0c55 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -45,20 +46,19 @@ class MixcloudIE(InfoExtractor):
         },
     }]
 
-    def _get_url(self, track_id, template_url):
-        server_count = 30
-        for i in range(server_count):
-            url = template_url % i
+    def _get_url(self, track_id, template_url, server_number):
+        boundaries = (1, 30)
+        for nr in server_numbers(server_number, boundaries):
+            url = template_url % nr
             try:
                 # We only want to know if the request succeed
                 # don't download the whole file
                 self._request_webpage(
                     HEADRequest(url), track_id,
-                    'Checking URL %d/%d ...' % (i + 1, server_count + 1))
+                    'Checking URL %d/%d ...' % (nr, boundaries[-1]))
                 return url
             except ExtractorError:
                 pass
-
         return None
 
     def _real_extract(self, url):
@@ -72,12 +72,13 @@ class MixcloudIE(InfoExtractor):
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
+        server_number = int(self._search_regex(r'stream(\d+)', song_url, 'server number'))
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
-        final_song_url = self._get_url(track_id, template_url)
+        final_song_url = self._get_url(track_id, template_url, server_number)
         if final_song_url is None:
             self.to_screen('Trying with m4a extension')
             template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            final_song_url = self._get_url(track_id, template_url)
+            final_song_url = self._get_url(track_id, template_url, server_number)
         if final_song_url is None:
             raise ExtractorError('Unable to extract track url')
 
@@ -114,3 +115,35 @@ class MixcloudIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
         }
+
+
+def server_numbers(first, boundaries):
+    """ Server numbers to try in descending order of probable availability.
+    Starting from first (i.e. the number of the server hosting the preview file)
+    and going further and further up to the higher boundary and down to the
+    lower one in an alternating fashion. Namely:
+
+        server_numbers(2, (1, 5))
+
+        # Where the preview server is 2, min number is 1 and max is 5.
+        # Yields: 2, 3, 1, 4, 5
+
+    Why not random numbers or increasing sequences? Since from what I've seen,
+    full length files seem to be hosted on servers whose number is closer to
+    that of the preview; to be confirmed.
+    """
+    zip_longest = getattr(itertools, 'zip_longest', None)
+    if zip_longest is None:
+        # python 2.x
+        zip_longest = itertools.izip_longest
+
+    if len(boundaries) != 2:
+        raise ValueError("boundaries should be a two-element tuple")
+    min, max = boundaries
+    highs = range(first + 1, max + 1)
+    lows = range(first - 1, min - 1, -1)
+    rest = filter(
+        None, itertools.chain.from_iterable(zip_longest(highs, lows)))
+    yield first
+    for n in rest:
+        yield n

From 3073a6d5e9036b0b613f57bc08099862a2af87f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 23:08:18 +0600
Subject: [PATCH 0031/1515] [ultimedia] Add extractor

Sponsored by thankyoumotion.com
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/ultimedia.py | 91 +++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/ultimedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 73c17aa84..867e7c935 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,6 +534,7 @@ from .twitch import (
     TwitchStreamIE,
 )
 from .ubu import UbuIE
+from .ultimedia import UltimediaIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
new file mode 100644
index 000000000..97e4445d4
--- /dev/null
+++ b/youtube_dl/extractor/ultimedia.py
@@ -0,0 +1,91 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    unified_strdate,
+    clean_html,
+)
+
+
+class UltimediaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/default/index/video[^/]+/id/(?P<id>[\d+a-z]+)'
+    _TESTS = [{
+        # news
+        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
+        'md5': '276a0e49de58c7e85d32b057837952a2',
+        'info_dict': {
+            'id': 's8uk0r',
+            'ext': 'mp4',
+            'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
+            'description': 'md5:3e5c8fd65791487333dda5db8aed32af',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150317',
+        },
+    }, {
+        # music
+        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
+        'md5': '2ea3513813cf230605c7e2ffe7eca61c',
+        'info_dict': {
+            'id': 'xvpfp8',
+            'ext': 'mp4',
+            'title': "Two - C'est la vie (Clip)",
+            'description': 'Two',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150224',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        deliver_url = self._search_regex(
+            r'<iframe[^>]+src="(https?://(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
+            webpage, 'deliver URL')
+
+        deliver_page = self._download_webpage(
+            deliver_url, video_id, 'Downloading iframe page')
+
+        player = self._parse_json(
+            self._search_regex(
+                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),
+            video_id)
+
+        quality = qualities(['flash', 'html5'])
+
+        formats = [{
+            'url': mode['config']['file'],
+            'format_id': mode.get('type'),
+            'quality': quality(mode.get('type')),
+        } for mode in player['modes']]
+        self._sort_formats(formats)
+
+        thumbnail = player.get('image')
+
+        title = clean_html((
+            self._html_search_regex(
+                r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
+                webpage, 'title', default=None)
+            or self._search_regex(
+                r"var\s+nameVideo\s*=\s*'([^']+)'",
+                deliver_page, 'title')))
+
+        description = clean_html(self._html_search_regex(
+            r'(?s)<span>Description</span>(.+?)</p>', webpage,
+            'description', fatal=False))
+
+        upload_date = unified_strdate(self._search_regex(
+            r'Ajouté le\s*<span>([^<]+)', webpage,
+            'upload date', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From d1dc7e39918ddfc3402a8ffd669e6c84ac971803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 23:11:48 +0600
Subject: [PATCH 0032/1515] [ultimedia] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 867e7c935..7eb9b4fbb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,11 +534,11 @@ from .twitch import (
     TwitchStreamIE,
 )
 from .ubu import UbuIE
-from .ultimedia import UltimediaIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
 )
+from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE

From 73900846b16d33f769d35ac945a97a66cc17fd5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 00:53:26 +0600
Subject: [PATCH 0033/1515] [ultimedia] Capture and output unavailable video
 message

---
 youtube_dl/extractor/ultimedia.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 97e4445d4..0c1b08d7d 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     qualities,
     unified_strdate,
     clean_html,
@@ -49,6 +50,10 @@ class UltimediaIE(InfoExtractor):
         deliver_page = self._download_webpage(
             deliver_url, video_id, 'Downloading iframe page')
 
+        if '>This video is currently not available' in deliver_page:
+            raise ExtractorError(
+                'Video %s is currently not available' % video_id, expected=True)
+
         player = self._parse_json(
             self._search_regex(
                 r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),

From b30ef07c6ccb982cff623c34e7c5cec5d8eb9bb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 01:06:39 +0600
Subject: [PATCH 0034/1515] [ultimedia] Handle youtube embeds

---
 youtube_dl/extractor/ultimedia.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 0c1b08d7d..06554a1be 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -60,12 +62,18 @@ class UltimediaIE(InfoExtractor):
             video_id)
 
         quality = qualities(['flash', 'html5'])
-
-        formats = [{
-            'url': mode['config']['file'],
-            'format_id': mode.get('type'),
-            'quality': quality(mode.get('type')),
-        } for mode in player['modes']]
+        formats = []
+        for mode in player['modes']:
+            video_url = mode.get('config', {}).get('file')
+            if not video_url:
+                continue
+            if re.match(r'https?://www\.youtube\.com/.+?', video_url):
+                return self.url_result(video_url, 'Youtube')
+            formats.append({
+                'url': video_url,
+                'format_id': mode.get('type'),
+                'quality': quality(mode.get('type')),
+            })
         self._sort_formats(formats)
 
         thumbnail = player.get('image')

From cbc3cfcab41599f1b52e328878e19d15be1792d4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Mar 2015 22:02:39 +0100
Subject: [PATCH 0035/1515] release 2015.03.18

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d6a1e67c6..72b365305 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -112,6 +112,7 @@
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
+ - **DouyuTV**
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
@@ -342,6 +343,7 @@
  - **PornHubPlaylist**
  - **Pornotube**
  - **PornoXO**
+ - **PrimeShareTV**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
@@ -367,6 +369,7 @@
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
+ - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
  - **RUHD**
  - **rutube**: Rutube videos
@@ -487,6 +490,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
+ - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
  - **ustream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ed07c375..51b4260aa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.15'
+__version__ = '2015.03.18'

From 0ae8bbac2d12a883b8eb7a941c65c7b87987d213 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:17:04 +0600
Subject: [PATCH 0036/1515] [nytimes] Support embed URL

---
 youtube_dl/extractor/nytimes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 56e1cad3b..5b57e95b6 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 
 class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/video/(?:[^/]+/)+(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',

From f3c0c667a6d8d1b58ca198b72a353887ea8c4f38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:23:52 +0600
Subject: [PATCH 0037/1515] [nytimes] Modernize

---
 youtube_dl/extractor/nytimes.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 5b57e95b6..d1cf8f4f3 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class NYTimesIE(InfoExtractor):
@@ -25,15 +27,15 @@ class NYTimesIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         video_data = self._download_json(
-            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id, video_id, 'Downloading video JSON')
+            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
+            video_id, 'Downloading video JSON')
 
         title = video_data['headline']
-        description = video_data['summary']
-        duration = video_data['duration'] / 1000.0
+        description = video_data.get('summary')
+        duration = float_or_none(video_data.get('duration'), 1000)
 
         uploader = video_data['byline']
         timestamp = parse_iso8601(video_data['publication_date'][:-8])
@@ -49,11 +51,11 @@ class NYTimesIE(InfoExtractor):
         formats = [
             {
                 'url': video['url'],
-                'format_id': video['type'],
-                'vcodec': video['video_codec'],
-                'width': video['width'],
-                'height': video['height'],
-                'filesize': get_file_size(video['fileSize']),
+                'format_id': video.get('type'),
+                'vcodec': video.get('video_codec'),
+                'width': int_or_none(video.get('width')),
+                'height': int_or_none(video.get('height')),
+                'filesize': get_file_size(video.get('fileSize')),
             } for video in video_data['renditions']
         ]
         self._sort_formats(formats)
@@ -61,7 +63,8 @@ class NYTimesIE(InfoExtractor):
         thumbnails = [
             {
                 'url': 'http://www.nytimes.com/%s' % image['url'],
-                'resolution': '%dx%d' % (image['width'], image['height']),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
             } for image in video_data['images']
         ]
 

From 3378d67a18d2d06574698168120bf8c7f7b9e172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:26:57 +0600
Subject: [PATCH 0038/1515] [generic] Add support for nytimes embeds (Closes
 #5234)

---
 youtube_dl/extractor/generic.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc5755d12..8716e4503 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1006,6 +1006,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for NYTimes player
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or

From eecc0685c976abc8fdcebe24e5c3a76d89268c54 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 19 Mar 2015 19:37:39 +0200
Subject: [PATCH 0039/1515] [videomega] Fix extraction and update test (Fixes
 #5235)

---
 youtube_dl/extractor/videomega.py | 45 ++++++++++---------------------
 1 file changed, 14 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 273030316..eb309a7cd 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,28 +4,21 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-    remove_start,
-)
+from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:www\.)?videomega\.tv/
-        (?:iframe\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
+        (?:iframe\.php|cdn\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
         '''
     _TEST = {
-        'url': 'http://videomega.tv/?ref=QR0HCUHI1661IHUCH0RQ',
+        'url': 'http://videomega.tv/?ref=4GNA688SU99US886ANG4',
         'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
         'info_dict': {
-            'id': 'QR0HCUHI1661IHUCH0RQ',
+            'id': '4GNA688SU99US886ANG4',
             'ext': 'mp4',
-            'title': 'Big Buck Bunny',
+            'title': 'BigBuckBunny_320x180',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -33,34 +26,24 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        iframe_url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
+        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
         req = compat_urllib_request.Request(iframe_url)
         req.add_header('Referer', url)
         webpage = self._download_webpage(req, video_id)
 
-        try:
-            escaped_data = re.findall(r'unescape\("([^"]+)"\)', webpage)[-1]
-        except IndexError:
-            raise ExtractorError('Unable to extract escaped data')
-
-        playlist = compat_urllib_parse.unquote(escaped_data)
-
+        title = self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'title')
+        title = re.sub(
+            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s?|\s?-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
-            r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
-        video_url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
-        title = remove_start(self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title'), 'VideoMega.tv - ')
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
-        self._sort_formats(formats)
+            r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+        video_url = self._search_regex(
+            r'<source[^>]+?src="([^"]+)"', webpage, 'video URL')
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
+            'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
                 'Referer': iframe_url,

From ccf3960eeca3ca8b75da709c8c3019531101f1e7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 19 Mar 2015 20:55:05 +0200
Subject: [PATCH 0040/1515] [nytimes] Improve _VALID_URL (Fixes #5238)

---
 youtube_dl/extractor/nytimes.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index d1cf8f4f3..03f0a4de6 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 
 class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
         'md5': '18a525a510f942ada2720db5f31644c0',
         'info_dict': {
@@ -24,7 +24,10 @@ class NYTimesIE(InfoExtractor):
             'uploader': 'Brett Weiner',
             'duration': 419,
         }
-    }
+    }, {
+        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 2684871bc10fe2b34f58312b6f75d3237a786732 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 20 Mar 2015 01:50:36 +0200
Subject: [PATCH 0041/1515] [vine] Fix formats extraction (Closes #5239)

---
 youtube_dl/extractor/vine.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 0b58fe0fe..c3187cfeb 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -33,14 +33,13 @@ class VineIE(InfoExtractor):
             r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
 
         formats = [{
-            'url': data['videoLowURL'],
-            'ext': 'mp4',
-            'format_id': 'low',
-        }, {
-            'url': data['videoUrl'],
-            'ext': 'mp4',
-            'format_id': 'standard',
-        }]
+            'format_id': '%(format)s-%(rate)s' % f,
+            'vcodec': f['format'],
+            'quality': f['rate'],
+            'url': f['videoUrl'],
+        } for f in data['videoUrls'] if f.get('rate')]
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 01218f919b3bb5d85449806ea832b29251dc13ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 14:59:38 +0100
Subject: [PATCH 0042/1515] [test/http] Add test for proxy support

---
 test/test_http.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/test/test_http.py b/test/test_http.py
index bd4d46fef..f2e305b6f 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,7 +8,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_http_server
+from youtube_dl.compat import compat_http_server, compat_urllib_request
 import ssl
 import threading
 
@@ -68,5 +68,52 @@ class TestHTTP(unittest.TestCase):
         r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
         self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)
 
+
+def _build_proxy_handler(name):
+    class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+        proxy_name = name
+
+        def log_message(self, format, *args):
+            pass
+
+        def do_GET(self):
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain; charset=utf-8')
+            self.end_headers()
+            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
+    return HTTPTestRequestHandler
+
+
+class TestProxy(unittest.TestCase):
+    def setUp(self):
+        self.proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('normal'))
+        self.port = self.proxy.socket.getsockname()[1]
+        self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
+        self.proxy_thread.daemon = True
+        self.proxy_thread.start()
+
+        self.cn_proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('cn'))
+        self.cn_port = self.cn_proxy.socket.getsockname()[1]
+        self.cn_proxy_thread = threading.Thread(target=self.cn_proxy.serve_forever)
+        self.cn_proxy_thread.daemon = True
+        self.cn_proxy_thread.start()
+
+    def test_proxy(self):
+        cn_proxy = 'localhost:{0}'.format(self.cn_port)
+        ydl = YoutubeDL({
+            'proxy': 'localhost:{0}'.format(self.port),
+            'cn_verification_proxy': cn_proxy,
+        })
+        url = 'http://foo.com/bar'
+        response = ydl.urlopen(url).read().decode('utf-8')
+        self.assertEqual(response, 'normal: {0}'.format(url))
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Ytdl-request-proxy', cn_proxy)
+        response = ydl.urlopen(req).read().decode('utf-8')
+        self.assertEqual(response, 'cn: {0}'.format(url))
+
 if __name__ == '__main__':
     unittest.main()

From f20bf146e217731016b32af598b177952fa1340a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 15:14:25 +0100
Subject: [PATCH 0043/1515] [test/YoutubeDL] split in two classes

The name was misleading
---
 test/test_YoutubeDL.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index db8a47d2d..2597a1993 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -339,6 +339,8 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+
+class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
         def s_formats(lang, autocaption=False):
             return [{

From 1c9a1457fc154697b327a87d9afe674425740287 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Mar 2015 20:53:14 +0600
Subject: [PATCH 0044/1515] [niconico] Add nm video test

---
 youtube_dl/extractor/niconico.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 4d57ac703..ddec7b338 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -22,7 +22,7 @@ class NiconicoIE(InfoExtractor):
     IE_NAME = 'niconico'
     IE_DESC = 'ニコニコ動画'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nicovideo.jp/watch/sm22312215',
         'md5': 'd1a75c0823e2f629128c43e1212760f9',
         'info_dict': {
@@ -39,7 +39,24 @@ class NiconicoIE(InfoExtractor):
             'username': 'ydl.niconico@gmail.com',
             'password': 'youtube-dl',
         },
-    }
+    }, {
+        'url': 'http://www.nicovideo.jp/watch/nm14296458',
+        'md5': '8db08e0158457cf852a31519fceea5bc',
+        'info_dict': {
+            'id': 'nm14296458',
+            'ext': 'swf',
+            'title': '【鏡音リン】Dance on media【オリジナル】take2!',
+            'description': 'md5:',
+            'uploader': 'りょうた',
+            'uploader_id': '18822557',
+            'upload_date': '20110429',
+            'duration': 209,
+        },
+        'params': {
+            'username': 'ydl.niconico@gmail.com',
+            'password': 'youtube-dl',
+        },
+    }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'

From 1887ecd4d629f44370846d372155512ec29e1f27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Mar 2015 21:45:09 +0600
Subject: [PATCH 0045/1515] [twitch] Fix login

---
 youtube_dl/extractor/twitch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index cbdaf9c7a..aad2bf222 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -67,14 +68,14 @@ class TwitchBaseIE(InfoExtractor):
             'authenticity_token': authenticity_token,
             'redirect_on_login': '',
             'embed_form': 'false',
-            'mp_source_action': '',
+            'mp_source_action': 'login-button',
             'follow': '',
-            'user[login]': username,
-            'user[password]': password,
+            'login': username,
+            'password': password,
         }
 
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)

From 531980d89c37427c663152a525fd795dc0b040e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 17:05:28 +0100
Subject: [PATCH 0046/1515] [test/YoutubeDL] test match_filter

---
 test/test_YoutubeDL.py | 68 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 2597a1993..652519831 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -14,6 +14,7 @@ from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
+from youtube_dl.utils import match_filter_func
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -463,6 +464,73 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(audiofile)
 
+    def test_match_filter(self):
+        class FilterYDL(YDL):
+            def __init__(self, *args, **kwargs):
+                super(FilterYDL, self).__init__(*args, **kwargs)
+                self.params['simulate'] = True
+
+            def process_info(self, info_dict):
+                super(YDL, self).process_info(info_dict)
+
+            def _match_entry(self, info_dict, incomplete):
+                res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
+                if res is None:
+                    self.downloaded_info_dicts.append(info_dict)
+                return res
+
+        first = {
+            'id': '1',
+            'url': TEST_URL,
+            'title': 'one',
+            'extractor': 'TEST',
+            'duration': 30,
+            'filesize': 10 * 1024,
+        }
+        second = {
+            'id': '2',
+            'url': TEST_URL,
+            'title': 'two',
+            'extractor': 'TEST',
+            'duration': 10,
+            'description': 'foo',
+            'filesize': 5 * 1024,
+        }
+        videos = [first, second]
+
+        def get_videos(filter_=None):
+            ydl = FilterYDL({'match_filter': filter_})
+            for v in videos:
+                ydl.process_ie_result(v, download=True)
+            return [v['id'] for v in ydl.downloaded_info_dicts]
+
+        res = get_videos()
+        self.assertEqual(res, ['1', '2'])
+
+        def f(v):
+            if v['id'] == '1':
+                return None
+            else:
+                return 'Video id is not 1'
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
+        f = match_filter_func('duration < 30')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('description = foo')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('description =? foo')
+        res = get_videos(f)
+        self.assertEqual(res, ['1', '2'])
+
+        f = match_filter_func('filesize > 5KiB')
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
 
 if __name__ == '__main__':
     unittest.main()

From a7d9ded45dac709fbb9140ab3525a10a2a6d3c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:07:23 +0100
Subject: [PATCH 0047/1515] [test] Add tests for aes

---
 devscripts/generate_aes_testdata.py | 36 ++++++++++++++++++++++
 test/test_aes.py                    | 47 +++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 devscripts/generate_aes_testdata.py
 create mode 100644 test/test_aes.py

diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
new file mode 100644
index 000000000..ff66449eb
--- /dev/null
+++ b/devscripts/generate_aes_testdata.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+import codecs
+import subprocess
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.utils import intlist_to_bytes
+from youtube_dl.aes import aes_encrypt, key_expansion
+
+secret_msg = b'Secret message goes here'
+
+
+def hex_str(int_list):
+    return codecs.encode(intlist_to_bytes(int_list), 'hex')
+
+
+def openssl_encode(algo, key, iv):
+    cmd = ['openssl', 'enc', '-e', '-' + algo, '-K', hex_str(key), '-iv', hex_str(iv)]
+    prog = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    out, _ = prog.communicate(secret_msg)
+    return out
+
+iv = key = [0x20, 0x15] + 14 * [0]
+
+r = openssl_encode('aes-128-cbc', key, iv)
+print('aes_cbc_decrypt')
+print(repr(r))
+
+password = key
+new_key = aes_encrypt(password, key_expansion(password))
+r = openssl_encode('aes-128-ctr', new_key, iv)
+print('aes_decrypt_text')
+print(repr(r))
diff --git a/test/test_aes.py b/test/test_aes.py
new file mode 100644
index 000000000..111b902e1
--- /dev/null
+++ b/test/test_aes.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_decrypt_text
+from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
+import base64
+
+# the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'
+
+
+class TestAES(unittest.TestCase):
+    def setUp(self):
+        self.key = self.iv = [0x20, 0x15] + 14 * [0]
+        self.secret_msg = b'Secret message goes here'
+
+    def test_encrypt(self):
+        msg = b'message'
+        key = list(range(16))
+        encrypted = aes_encrypt(bytes_to_intlist(msg), key)
+        decrypted = intlist_to_bytes(aes_decrypt(encrypted, key))
+        self.assertEqual(decrypted, msg)
+
+    def test_cbc_decrypt(self):
+        data = bytes_to_intlist(
+            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd"
+        )
+        decrypted = intlist_to_bytes(aes_cbc_decrypt(data, self.key, self.iv))
+        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+
+    def test_decrypt_text(self):
+        password = intlist_to_bytes(self.key).decode('utf-8')
+        encrypted = base64.b64encode(
+            intlist_to_bytes(self.iv[:8]) +
+            b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
+        )
+        decrypted = (aes_decrypt_text(encrypted, password, 16))
+        self.assertEqual(decrypted, self.secret_msg)
+
+if __name__ == '__main__':
+    unittest.main()

From 85698c508651cc047b0ba2181580d99ed057ef0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:18:33 +0100
Subject: [PATCH 0048/1515] [crunchyroll] Remove unused class

---
 youtube_dl/extractor/crunchyroll.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index e64b88fbc..6ded723c9 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -23,7 +23,6 @@ from ..utils import (
 )
 from ..aes import (
     aes_cbc_decrypt,
-    inc,
 )
 
 
@@ -102,13 +101,6 @@ class CrunchyrollIE(InfoExtractor):
 
         key = obfuscate_key(id)
 
-        class Counter:
-            __value = iv
-
-            def next_value(self):
-                temp = self.__value
-                self.__value = inc(self.__value)
-                return temp
         decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
         return zlib.decompress(decrypted_data)
 

From 179d6678b1ce5ecdf22e2ae254f451183d836e75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:34:44 +0100
Subject: [PATCH 0049/1515] Remove the 'stitle' field

A warning has been printed for more than 2 years (since 97cd3afc7525394c46398f1526d412d081c02085)
---
 youtube_dl/YoutubeDL.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a83bc956..b5ef5e009 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -328,9 +328,6 @@ class YoutubeDL(object):
                 'Parameter outtmpl is bytes, but should be a unicode string. '
                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 
-        if '%(stitle)s' in self.params.get('outtmpl', ''):
-            self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
-
         self._setup_opener()
 
         if auto_init:
@@ -1218,9 +1215,6 @@ class YoutubeDL(object):
         if len(info_dict['title']) > 200:
             info_dict['title'] = info_dict['title'][:197] + '...'
 
-        # Keep for backwards compatibility
-        info_dict['stitle'] = info_dict['title']
-
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 

From 4e6a2286899ab156ad342e3e9003b0c3b239e0ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:20:49 +0600
Subject: [PATCH 0050/1515] [nrk] Adapt to new URL format

---
 youtube_dl/extractor/nrk.py | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index bff36f9d3..d17b7ed49 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -14,22 +14,22 @@ from ..utils import (
 
 
 class NRKIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:video|lyd)/[^/]+/(?P<id>[\dA-F]{16})'
+    _VALID_URL = r'(?:nrk:|http://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
 
     _TESTS = [
         {
-            'url': 'http://www.nrk.no/video/dompap_og_andre_fugler_i_piip_show/D0FA54B5C8B6CE59/emne/piipshow/',
-            'md5': 'a6eac35052f3b242bb6bb7f43aed5886',
+            'url': 'http://www.nrk.no/video/PS*150533',
+            'md5': 'bccd850baebefe23b56d708a113229c2',
             'info_dict': {
                 'id': '150533',
                 'ext': 'flv',
                 'title': 'Dompap og andre fugler i Piip-Show',
-                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f'
+                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
             }
         },
         {
-            'url': 'http://www.nrk.no/lyd/lyd_av_oppleser_for_blinde/AEFDDD5473BA0198/',
-            'md5': '3471f2a51718195164e88f46bf427668',
+            'url': 'http://www.nrk.no/video/PS*154915',
+            'md5': '0b1493ba1aae7d9579a5ad5531bc395a',
             'info_dict': {
                 'id': '154915',
                 'ext': 'flv',
@@ -40,20 +40,18 @@ class NRKIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage(url, video_id)
-
-        video_id = self._html_search_regex(r'<div class="nrk-video" data-nrk-id="(\d+)">', page, 'video id')
+        video_id = self._match_id(url)
 
         data = self._download_json(
-            'http://v7.psapi.nrk.no/mediaelement/%s' % video_id, video_id, 'Downloading media JSON')
+            'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
+            video_id, 'Downloading media JSON')
 
         if data['usageRights']['isGeoBlocked']:
-            raise ExtractorError('NRK har ikke rettig-heter til å vise dette programmet utenfor Norge', expected=True)
+            raise ExtractorError(
+                'NRK har ikke rettig-heter til å vise dette programmet utenfor Norge',
+                expected=True)
 
-        video_url = data['mediaUrl'] + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124'
+        video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
 
         images = data.get('images')
         if images:

From 393d9fc6d28704ef7b60acb1dfc785493ecbe0eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:21:19 +0600
Subject: [PATCH 0051/1515] [nrk] Extract duration

---
 youtube_dl/extractor/nrk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d17b7ed49..117a78aa1 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -25,6 +25,7 @@ class NRKIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Dompap og andre fugler i Piip-Show',
                 'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
+                'duration': 263,
             }
         },
         {
@@ -35,6 +36,7 @@ class NRKIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Slik høres internett ut når du er blind',
                 'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
+                'duration': 20,
             }
         },
     ]
@@ -53,6 +55,8 @@ class NRKIE(InfoExtractor):
 
         video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
 
+        duration = parse_duration(data.get('duration'))
+
         images = data.get('images')
         if images:
             thumbnails = images['webImages']
@@ -67,6 +71,7 @@ class NRKIE(InfoExtractor):
             'ext': 'flv',
             'title': data['title'],
             'description': data['description'],
+            'duration': duration,
             'thumbnail': thumbnail,
         }
 

From faa1b5c2920f1d50f01b83e96b5b0f136edc74f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:22:08 +0600
Subject: [PATCH 0052/1515] [nrk:playlist] Add extractor (Closes #5245)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nrk.py      | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7eb9b4fbb..e25e4a582 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -346,6 +346,7 @@ from .npo import (
 )
 from .nrk import (
     NRKIE,
+    NRKPlaylistIE,
     NRKTVIE,
 )
 from .ntvde import NTVDeIE
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 117a78aa1..c2a8202dd 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -76,6 +76,37 @@ class NRKIE(InfoExtractor):
         }
 
 
+class NRKPlaylistIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:[^/]+/)*(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
+        'info_dict': {
+            'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
+            'title': 'Gjenopplev den historiske solformørkelsen',
+            'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
+        },
+        'playlist_mincount': 2,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('nrk:%s' % video_id, 'NRK')
+            for video_id in re.findall(
+                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="(\d+)"', webpage)
+        ]
+
+        playlist_title = self._og_search_title(webpage)
+        playlist_description = self._og_search_description(webpage)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_title, playlist_description)
+
+
 class NRKTVIE(InfoExtractor):
     _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 

From c9450c7ab19bbd24a4ff8643370cb4f235b0b380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 14:00:37 +0100
Subject: [PATCH 0053/1515] [nrk:playlist] Restrict _VALID_URL

It would also match /videos/PS... urls
---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index c2a8202dd..1355ecea2 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,7 +77,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)[^/]+/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',

From 5379a2d40db3674fead7c4239afdb65ff7b389c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 14:12:43 +0100
Subject: [PATCH 0054/1515] [test/utils] Test xpath_text

---
 test/test_utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3431ad24e..a8ab87685 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -24,6 +24,7 @@ from youtube_dl.utils import (
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
     InAdvancePagedList,
@@ -54,6 +55,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    xpath_text,
     render_table,
     match_str,
 )
@@ -250,6 +252,17 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find('media:song/media:author').text, 'The Author')
         self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
 
+    def test_xpath_text(self):
+        testxml = '''<root>
+            <div>
+                <p>Foo</p>
+            </div>
+        </root>'''
+        doc = xml.etree.ElementTree.fromstring(testxml)
+        self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
+        self.assertTrue(xpath_text(doc, 'div/bar') is None)
+        self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True)
+
     def test_smuggle_url(self):
         data = {"ö": "ö", "abc": [3]}
         url = 'https://foo.bar/baz?x=y#a'

From a09141548aa31db7c7d9457b10f5c84e6e32beba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 20:42:48 +0600
Subject: [PATCH 0055/1515] [nrk:playlist] Relax video id regex and improve
 _VALID_URL

---
 youtube_dl/extractor/nrk.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 1355ecea2..e91d3a248 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,17 +77,25 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
         'info_dict': {
             'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
             'title': 'Gjenopplev den historiske solformørkelsen',
             'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
         },
-        'playlist_mincount': 2,
-    }
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
+        'info_dict': {
+            'id': 'rivertonprisen-til-karin-fossum-1.12266449',
+            'title': 'Rivertonprisen til Karin Fossum',
+            'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
+        },
+        'playlist_count': 5,
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -97,7 +105,8 @@ class NRKPlaylistIE(InfoExtractor):
         entries = [
             self.url_result('nrk:%s' % video_id, 'NRK')
             for video_id in re.findall(
-                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="(\d+)"', webpage)
+                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"',
+                webpage)
         ]
 
         playlist_title = self._og_search_title(webpage)

From 49aeedb8cb4dcf317c970a58c590d42e37904720 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:11:10 +0600
Subject: [PATCH 0056/1515] [libsyn] Improve and simplify

---
 youtube_dl/extractor/libsyn.py | 81 +++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 88379f276..6bf741db8 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -1,50 +1,59 @@
-# encoding: utf-8
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
+from ..utils import unified_strdate
+
 
 class LibsynIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
-    _TESTS = [{
-        'url': "http://html5-player.libsyn.com/embed/episode/id/3377616/",
+    _VALID_URL = r'https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://html5-player.libsyn.com/embed/episode/id/3377616/',
+        'md5': '443360ee1b58007bc3dcf09b41d093bb',
         'info_dict': {
-            'id': "3377616",
-            'ext': "mp3",
-            'title': "Episode 12: Bassem Youssef: Egypt's Jon Stewart",
-            'description': "<p>Bassem Youssef joins executive producer Steve Bodow and senior producer Sara Taksler for a conversation about how&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">The Daily Show</em>&nbsp;inspired Bassem to create&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Al-Bernameg</em>, his massively popular (and now banned) Egyptian news satire program. Sara discusses her soon-to-be-released documentary,&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Tickling Giants</em>, which chronicles how Bassem and his staff risked their safety every day to tell jokes.</p>",
+            'id': '3377616',
+            'ext': 'mp3',
+            'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+            'description': 'md5:601cb790edd05908957dae8aaa866465',
+            'upload_date': '20150220',
         },
-    }]
+    }
 
     def _real_extract(self, url):
-        if url.startswith('//'):
-            url = 'https:' + url
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        video_id = self._match_id(url)
 
-        podcast_title         = self._search_regex(r'<h2>(.*?)</h2>', webpage, 'show title')
-        podcast_episode_title = self._search_regex(r'<h3>(.*?)</h3>', webpage, 'episode title')
-        podcast_date          = unified_strdate(self._search_regex(r'<div class="release_date">Released: (.*?)</div>', webpage, 'release date'))
-        podcast_description   = self._search_regex(r'<div id="info_text_body">(.*?)</div>', webpage, 'description')
+        webpage = self._download_webpage(url, video_id)
 
-        url0 = self._search_regex(r'var mediaURLLibsyn = "(?P<url0>https?://.*)";', webpage, 'first media URL')
-        url1 = self._search_regex(r'var mediaURL = "(?P<url1>https?://.*)";', webpage, 'second media URL')
+        formats = [{
+            'url': media_url,
+        } for media_url in set(re.findall('var\s+mediaURL(?:Libsyn)?\s*=\s*"([^"]+)"', webpage))]
 
-        if url0 != url1:
-            formats = [{
-                'url': url0
-            }, {
-                'url': url1
-            }]
-        else:
-            formats = [{
-                'url': url0
-            }]
+        podcast_title = self._search_regex(
+            r'<h2>([^<]+)</h2>', webpage, 'title')
+        episode_title = self._search_regex(
+            r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
+
+        title = '%s - %s' %(podcast_title, episode_title) if podcast_title else episode_title
+
+        description = self._html_search_regex(
+            r'<div id="info_text_body">(.+?)</div>', webpage,
+            'description', fatal=False)
+
+        thumbnail = self._search_regex(
+            r'<img[^>]+class="info-show-icon"[^>]+src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        release_date = unified_strdate(self._search_regex(
+            r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))
 
         return {
-            'id': display_id,
-            'title': podcast_episode_title,
-            'description': podcast_description,
-            'upload_date': podcast_date,
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': release_date,
             'formats': formats,
         }

From a1d0aa7b882484685a1a02185d0dafd51c545701 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:11:47 +0600
Subject: [PATCH 0057/1515] [libsyn] Fix extractor alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a20492fc3..82b75a144 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -250,8 +250,8 @@ from .letv import (
     LetvTvIE,
     LetvPlaylistIE
 )
-from .lifenews import LifeNewsIE
 from .libsyn import LibsynIE
+from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,

From cefdf970ccd8017cd67e949004e5e4c770aacdb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:18:13 +0600
Subject: [PATCH 0058/1515] [extractor/generic] Support Libsyn embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8716e4503..84e8f14b2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1013,6 +1013,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for Libsyn player
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or

From 2051acdeb2ed9a0edf3b6b70682699c37d19d851 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:20:27 +0600
Subject: [PATCH 0059/1515] [extractor/generic] Add test for Libsyn embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 84e8f14b2..8a49b0b54 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -527,6 +527,17 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Viddler'],
         },
+        # Libsyn embed
+        {
+            'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
+            'info_dict': {
+                'id': '3377616',
+                'ext': 'mp3',
+                'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+                'description': 'md5:601cb790edd05908957dae8aaa866465',
+                'upload_date': '20150220',
+            },
+        },
         # jwplayer YouTube
         {
             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',

From cf2e2eb1c0b626f2d5f210ffd14642aceb0358e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:23:20 +0600
Subject: [PATCH 0060/1515] [comedycentral] Drop thedailyshow podcast extractor

Generic extractor is just fine for Libsyn embeds
---
 youtube_dl/extractor/__init__.py      |  2 +-
 youtube_dl/extractor/comedycentral.py | 24 ------------------------
 2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 82b75a144..d73826d44 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -84,7 +84,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE, TheDailyShowPodcastIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index bd3817b56..648a6f990 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -273,27 +273,3 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
-
-class TheDailyShowPodcastIE(InfoExtractor):
-    _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
-    _TESTS = [{
-        "url": "http://thedailyshow.cc.com/podcast/episodetwelve",
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        player_url = self._search_regex(r'<iframe(?:\s+[^>]+)?\s*src="((?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/[0-9]+)', webpage, 'player URL')
-        if player_url.startswith('//'):
-            mobj = re.match(self._VALID_URL, url)
-            scheme = mobj.group('scheme')
-            if not scheme:
-                scheme = 'https:'
-            player_url = scheme + player_url
-
-        return {
-            '_type': 'url_transparent',
-            'url': player_url,
-        }

From 1a4123de04d0168ef4a14e6064148eb248d65dc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:23:38 +0600
Subject: [PATCH 0061/1515] [comedycentral] Remove unused import

---
 youtube_dl/extractor/comedycentral.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 648a6f990..e5edcc84b 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,

From 336d19044c84128cc3bdcb62882372243dfa32c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 11:03:52 +0100
Subject: [PATCH 0062/1515] [lybsyn] pep8: add space around operator

---
 youtube_dl/extractor/libsyn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 6bf741db8..9ab1416f5 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -36,7 +36,7 @@ class LibsynIE(InfoExtractor):
         episode_title = self._search_regex(
             r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
 
-        title = '%s - %s' %(podcast_title, episode_title) if podcast_title else episode_title
+        title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title
 
         description = self._html_search_regex(
             r'<div id="info_text_body">(.+?)</div>', webpage,

From f9544f6e8fe4697a4a48364b6d7a7dc7d93cabf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 12:09:58 +0100
Subject: [PATCH 0063/1515] [test/aes] Test aes_decrypt_text with 256 bit

---
 devscripts/generate_aes_testdata.py | 8 +++++++-
 test/test_aes.py                    | 8 ++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
index ff66449eb..2e389fc8e 100644
--- a/devscripts/generate_aes_testdata.py
+++ b/devscripts/generate_aes_testdata.py
@@ -32,5 +32,11 @@ print(repr(r))
 password = key
 new_key = aes_encrypt(password, key_expansion(password))
 r = openssl_encode('aes-128-ctr', new_key, iv)
-print('aes_decrypt_text')
+print('aes_decrypt_text 16')
+print(repr(r))
+
+password = key + 16 * [0]
+new_key = aes_encrypt(password, key_expansion(password)) * (32 // 16)
+r = openssl_encode('aes-256-ctr', new_key, iv)
+print('aes_decrypt_text 32')
 print(repr(r))
diff --git a/test/test_aes.py b/test/test_aes.py
index 111b902e1..4dc7de7b5 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -43,5 +43,13 @@ class TestAES(unittest.TestCase):
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
+        password = intlist_to_bytes(self.key).decode('utf-8')
+        encrypted = base64.b64encode(
+            intlist_to_bytes(self.iv[:8]) +
+            b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
+        )
+        decrypted = (aes_decrypt_text(encrypted, password, 32))
+        self.assertEqual(decrypted, self.secret_msg)
+
 if __name__ == '__main__':
     unittest.main()

From 93f787070ff2de48023c5bfeaebd153b3e29137a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 15:39:35 +0100
Subject: [PATCH 0064/1515] [twitch] Only match digits for the video id

Urls can also contain contain a query (for example a timestamp '?t=foo')
---
 youtube_dl/extractor/twitch.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index aad2bf222..94bd6345d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -149,7 +149,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
 
 class TwitchVideoIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:video'
-    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'video'
     _ITEM_SHORTCUT = 'a'
 
@@ -165,7 +165,7 @@ class TwitchVideoIE(TwitchItemBaseIE):
 
 class TwitchChapterIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:chapter'
-    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
@@ -184,7 +184,7 @@ class TwitchChapterIE(TwitchItemBaseIE):
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'
-    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 

From 32d687f55e103963a2cb8d8f3f88bb31b9cb8fb6 Mon Sep 17 00:00:00 2001
From: zx8 <zx8@mac>
Date: Sun, 22 Mar 2015 18:03:40 +0000
Subject: [PATCH 0065/1515] [safari] Add safaribooksonline extractor

---
 youtube_dl/extractor/__init__.py |   4 +
 youtube_dl/extractor/safari.py   | 144 +++++++++++++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 youtube_dl/extractor/safari.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d73826d44..3a0c42ded 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -420,6 +420,10 @@ from .rutube import (
 )
 from .rutv import RUTVIE
 from .sandia import SandiaIE
+from .safari import (
+    SafariIE,
+    SafariCourseIE,
+)
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
new file mode 100644
index 000000000..3e494b960
--- /dev/null
+++ b/youtube_dl/extractor/safari.py
@@ -0,0 +1,144 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    smuggle_url,
+    std_headers,
+)
+
+
+class SafariBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
+    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]+>Sign Out</a>'
+    _ACCOUNT_CREDENTIALS_HINT = ('Use --username and --password options to '
+                                 'supply credentials for safaribooksonline.com ')
+    _NETRC_MACHINE = 'safaribooksonline'
+
+    LOGGED_IN = False
+
+    def _real_initialize(self):
+        # We only need to log in once for courses or individual videos
+        if not SafariBaseIE.LOGGED_IN:
+            self._login()
+            SafariBaseIE.LOGGED_IN = True
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            raise ExtractorError(
+                self._ACCOUNT_CREDENTIALS_HINT,
+                expected=True)
+
+        headers = std_headers
+        if 'Referer' not in headers:
+            headers['Referer'] = self._LOGIN_URL
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None,
+            'Downloading login form')
+
+        csrf = self._html_search_regex(
+            r"<input +type='hidden' +name='csrfmiddlewaretoken' +value='([^']+)' +/>",
+            login_page, 'csrf token')
+
+        login_form = {
+            'csrfmiddlewaretoken': csrf,
+            'email': username,
+            'password1': password,
+            'login': 'Sign In',
+            'next': '',
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
+        login_page = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+            raise ExtractorError('Login failed; make sure your credentials are correct and '
+                                 'try again.', expected=True)
+
+        self.to_screen('Login successful')
+
+
+class SafariIE(SafariBaseIE):
+    IE_NAME = 'safari'
+    IE_DESC = 'safaribooksonline.com online video'
+    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/'
+                  '(?P<id>\d+)/(?P<part>part\d+)\.html')
+    _TEST = {
+        'url': ('https://www.safaribooksonline.com/library/view/'
+                'hadoop-fundamentals-livelessons/9780133392838/part00.html'),
+        'md5': '5b0c4cc1b3c1ba15dda7344085aa5592',
+        'info_dict': {
+            'id': '9780133392838',
+            'ext': 'mp4',
+            'title': 'Introduction',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        part = mobj.group('part')
+
+        webpage = self._download_webpage(url, part)
+        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+        if not bc_url:
+            raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
+
+        return {
+            '_type': 'url',
+            'url': smuggle_url(bc_url, {'Referer': url}),
+            'ie_key': 'Brightcove'
+        }
+
+
+class SafariCourseIE(SafariBaseIE):
+    IE_NAME = 'safari:course'
+    IE_DESC = 'safaribooksonline.com online courses'
+
+    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/'
+                  '(?P<course_path>[^/]+)/(?P<id>\d+)/?$')
+
+    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
+    _API_FORMAT = 'json'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_path = mobj.group('course_path')
+        course_id = mobj.group('id')
+
+        webpage = self._download_webpage(
+            '%s/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
+            course_path, 'Downloading course JSON')
+
+        course_json = json.loads(webpage)
+
+        if 'chapters' not in course_json:
+            raise ExtractorError('No chapters found for course %s' % course_id, expected=True)
+
+        num_parts = len(course_json['chapters'])
+        parts = ['%02d' % part for part in range(num_parts)]
+
+        entries = [
+            self.url_result(
+                'https://www.safaribooksonline.com/library/view/%s/%s/part%s.html' % (course_path,
+                                                                                      course_id,
+                                                                                      part_id),
+                'Safari')
+            for part_id in parts]
+
+        course_title = course_json['title']
+
+        return self.playlist_result(entries, course_id, course_title)

From 575dad3c9842f333c4af27563a26bddaf0015fa2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Sun, 22 Mar 2015 20:25:44 +0100
Subject: [PATCH 0066/1515] [pornovoisines] Add extractor

---
 youtube_dl/extractor/__init__.py      |   1 +
 youtube_dl/extractor/pornovoisines.py | 101 ++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/pornovoisines.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d73826d44..17d075ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -382,6 +382,7 @@ from .pornhub import (
     PornHubPlaylistIE,
 )
 from .pornotube import PornotubeIE
+from .pornovoisines import PornoVoisinesIE
 from .pornoxo import PornoXOIE
 from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
new file mode 100644
index 000000000..efbb6a818
--- /dev/null
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import datetime
+import random
+
+from ..compat import compat_urllib_parse
+from .common import InfoExtractor
+
+class PornoVoisinesIE(InfoExtractor):
+    _VALID_URL = r'^((?:http://)?(?:www\.)?pornovoisines.com)/showvideo/(\d+)/([^/]+)'
+
+    VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
+        '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
+
+    SERVER_NUMBERS = (1, 2)
+
+    _TEST = {
+        'url': 'http://www.pornovoisines.com/showvideo/1285/recherche-appartement/',
+        'md5': '5ac670803bc12e9e7f9f662ce64cf1d1',
+        'info_dict': {
+            'id': '1285',
+            'display_id': 'recherche-appartement',
+            'ext': 'mp4',
+            'title': "Recherche appartement",
+            'upload_date': '20140925',
+            'view_count': int,
+            'duration': 120,
+            'categories': ["Débutante", "Scénario", "Sodomie"],
+            'description': 're:^Pour la .+ original...$',
+            'thumbnail': 're:^http://',
+            'uploader': "JMTV",
+            'average_rating': float,
+            'comment_count': int,
+            'age_limit': 18,
+        }
+    }
+
+    @classmethod
+    def build_video_url(cls, id):
+        server_nr = random.choice(cls.SERVER_NUMBERS)
+        return cls.VIDEO_URL_TEMPLATE % (server_nr, id)
+
+    @staticmethod
+    def parse_upload_date(str):
+        return datetime.datetime.strptime(str, "%d-%m-%Y").strftime("%Y%m%d")
+
+    @staticmethod
+    def parse_categories(str):
+        return map(lambda s: s.strip(), str.split(','))
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        url_prefix = mobj.group(1)
+        id = mobj.group(2)
+        display_id = mobj.group(3)
+
+        webpage = self._download_webpage(url, id)
+
+        title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title',
+            flags=re.DOTALL)
+        url = self.build_video_url(id)
+        upload_date = self.parse_upload_date(
+            self._search_regex(r'Publié le (\d\d-\d\d-\d{4})', webpage,
+            'upload date'))
+        view_count = int(self._search_regex(r'(\d+) vues', webpage, 'view count'))
+        duration = int(self._search_regex('Durée (\d+)', webpage, 'duration'))
+        categories = self.parse_categories(self._html_search_regex(
+            r'<li class="categorie">(.+?)</li>', webpage, "categories",
+            flags=re.DOTALL))
+        description = self._html_search_regex(
+            r'<article id="descriptif">(.+?)</article>', webpage, "description",
+            flags=re.DOTALL)
+        thumbnail = url_prefix + self._html_search_regex(re.compile(
+            '<div id="mediaspace' + id + '">.*?<img src="(.+?)"', re.DOTALL),
+            webpage, "thumbnail")
+        uploader = re.sub(r' *\| *$', '',
+            self._html_search_regex(r'<li class="auteur">(.+?)</li>', webpage,
+            "uploader", flags=re.DOTALL))
+        average_rating = float(self._search_regex(r'Note : (\d+,\d+)',
+            webpage, "average rating").replace(',', '.'))
+        comment_count = int(self._search_regex(r'\((\d+)\)', webpage,
+            "comment count"))
+
+        return {
+            'id': id,
+            'display_id': display_id,
+            'url': url,
+            'title': title,
+            'upload_date': upload_date,
+            'view_count': view_count,
+            'duration': duration,
+            'categories': categories,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'average_rating': average_rating,
+            'comment_count': comment_count,
+            'age_limit': 18,
+        }

From c41a2ec4af9fa76b04b6d9f50d9a895d124ea14c Mon Sep 17 00:00:00 2001
From: tiktok <tiktok7@users.noreply.github.com>
Date: Mon, 23 Mar 2015 01:42:17 +0100
Subject: [PATCH 0067/1515] [MiomioTv] Add new extractor

---
 docs/supportedsites.md            |  1 +
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/miomio_tv.py | 70 +++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/miomio_tv.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 062cb3d62..53d280677 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -244,6 +244,7 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
+ - **Miomio.tv**
  - **mitele.es**
  - **mixcloud**
  - **MLB**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..370154773 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -265,6 +265,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .miomio_tv import MiomioTvIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio_tv.py
new file mode 100644
index 000000000..355774f54
--- /dev/null
+++ b/youtube_dl/extractor/miomio_tv.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MiomioTvIE(InfoExtractor):
+    IE_NAME = 'miomio.tv'
+    _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.miomio.tv/watch/cc179734/',
+        'md5': '48de02137d0739c15b440a224ad364b9',
+        'info_dict': {
+            'id': '179734',
+            'title': u'\u624b\u7ed8\u52a8\u6f2b\u9b3c\u6ce3\u4f46\u4e01\u5168\u7a0b\u753b\u6cd5',
+            'ext': 'flv'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<meta\s+name="description"\s+content="\s*([^"]*)\s*"', webpage, 'title')
+        ref_path = self._search_regex(r'src="(/mioplayer/.*?)"', webpage, 'ref_path')
+        referer = 'http://www.miomio.tv{}'.format(ref_path)
+        xml_config = self._search_regex(r'flashvars="type=sina&amp;(.*?)&amp;cid=', webpage, 'xml config')
+        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={}&r=cc{}".format(id, 945), video_id)
+        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{}'.format(xml_config)
+        vidconfig = self._download_xml(xml_url, video_id)
+
+        file_els = vidconfig.findall('.//durl')
+
+        entries = []
+
+        for file_el in file_els:
+            segment_id = file_el.find('order').text.strip()
+            segment_title = '_'.join([title, segment_id])
+            segment_duration = file_el.find('length').text.strip()
+            segment_url = file_el.find('url').text.strip()
+
+            entries.append({
+                'id': segment_id,
+                'title': segment_title,
+                'duration': segment_duration,
+                'url': segment_url
+            })
+
+        http_headers = {
+            'Referer': referer,
+            'Accept-Language': 'en,en-US;q=0.7,de;q=0.3',
+            'Accept-Encoding': 'gzip, deflate',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+        }
+
+        if len(entries) == 1:
+            return {
+                'id': video_id,
+                'title': title,
+                'url': entries[0]['url'],
+                'http_headers': http_headers
+            }
+
+        return {
+            '_type': 'multi_video',
+            'id': video_id,
+            'title': title,
+            'entries': entries,
+            'http_headers': http_headers
+        }

From 9f790b9901445d532564b3c5c90c0ecfaea4805d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Mar 2015 21:23:57 +0600
Subject: [PATCH 0068/1515] [mlb] Improve _VALID_URL (Closes #5260)

---
 youtube_dl/extractor/mlb.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 1a241aca7..e369551c2 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -80,6 +80,10 @@ class MLBIE(InfoExtractor):
             'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
             'only_matching': True,
         },
+        {
+            'url': 'http://m.cardinals.mlb.com/stl/video/v51175783/atlstl-piscotty-makes-great-sliding-catch-on-line/?partnerId=as_mlb_20150321_42500876&adbid=579409712979910656&adbpl=tw&adbpr=52847728',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From b0872c19ea6fb5dcc20d695d6faead4af42da364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Mar 2015 22:15:01 +0600
Subject: [PATCH 0069/1515] [npo] Skip broken URL links (Closes #5266)

---
 youtube_dl/extractor/npo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 557dffa46..5d8448571 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -231,7 +231,10 @@ class NPOLiveIE(NPOBaseIE):
                 stream_url = self._download_json(
                     stream_info['stream'], display_id,
                     'Downloading %s URL' % stream_type,
-                    transform_source=strip_jsonp)
+                    'Unable to download %s URL' % stream_type,
+                    transform_source=strip_jsonp, fatal=False)
+                if not stream_url:
+                    continue
                 if stream_type == 'hds':
                     f4m_formats = self._extract_f4m_formats(stream_url, display_id)
                     # f4m downloader downloads only piece of live stream

From 5d1f0e607b3dfe7b1adea09a1188011e57acf0fb Mon Sep 17 00:00:00 2001
From: tiktok <tiktok7@users.noreply.github.com>
Date: Mon, 23 Mar 2015 23:16:50 +0100
Subject: [PATCH 0070/1515] [MiomioTv] updated based on feedback to merge
 request:     1) added comment to explain extra xml link download     2)
 changed {} entries to {0}, {1} etc     3) removed redundant language header
 (the others are required)     4) checked out the old version of the supported
 sites md (the change was     not required)

---
 docs/supportedsites.md            |  1 -
 youtube_dl/extractor/miomio_tv.py | 12 ++++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 53d280677..062cb3d62 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -244,7 +244,6 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
- - **Miomio.tv**
  - **mitele.es**
  - **mixcloud**
  - **MLB**
diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio_tv.py
index 355774f54..ae20a32fa 100644
--- a/youtube_dl/extractor/miomio_tv.py
+++ b/youtube_dl/extractor/miomio_tv.py
@@ -23,10 +23,15 @@ class MiomioTvIE(InfoExtractor):
 
         title = self._html_search_regex(r'<meta\s+name="description"\s+content="\s*([^"]*)\s*"', webpage, 'title')
         ref_path = self._search_regex(r'src="(/mioplayer/.*?)"', webpage, 'ref_path')
-        referer = 'http://www.miomio.tv{}'.format(ref_path)
+        referer = 'http://www.miomio.tv{0}'.format(ref_path)
         xml_config = self._search_regex(r'flashvars="type=sina&amp;(.*?)&amp;cid=', webpage, 'xml config')
-        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={}&r=cc{}".format(id, 945), video_id)
-        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{}'.format(xml_config)
+        
+        # skipping the following page causes lags and eventually connection drop-outs
+        # id is normally a rotating three digit value but a fixed value always appears to work
+        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={0}&r=cc{1}".format(id, 945), video_id)
+
+        # the following xml contains the actual configuration information on the video file(s)
+        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config)
         vidconfig = self._download_xml(xml_url, video_id)
 
         file_els = vidconfig.findall('.//durl')
@@ -48,7 +53,6 @@ class MiomioTvIE(InfoExtractor):
 
         http_headers = {
             'Referer': referer,
-            'Accept-Language': 'en,en-US;q=0.7,de;q=0.3',
             'Accept-Encoding': 'gzip, deflate',
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
         }

From f5e2efbbf0a4a1fa02df09a12ce9ec69970ab14c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Mar 2015 16:39:46 +0100
Subject: [PATCH 0071/1515] [options] Handle special characters in argv (Fixes
 #5157)

---
 test/test_execution.py | 9 +++++++++
 youtube_dl/options.py  | 5 +++++
 2 files changed, 14 insertions(+)

diff --git a/test/test_execution.py b/test/test_execution.py
index 60df187de..f31e51558 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+# coding: utf-8
+
 from __future__ import unicode_literals
 
 import unittest
@@ -27,5 +29,12 @@ class TestExecution(unittest.TestCase):
     def test_main_exec(self):
         subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
+    def test_cmdline_umlauts(self):
+        p = subprocess.Popen(
+            [sys.executable, 'youtube_dl/__main__.py', 'ä', '--version'],
+            cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
+        _, stderr = p.communicate()
+        self.assertFalse(stderr)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 4e6e47d6f..35c7e5fb3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -794,6 +794,11 @@ def parseOpts(overrideArguments=None):
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
         command_line_conf = sys.argv[1:]
+        # Workaround for Python 2.x, where argv is a byte list
+        if sys.version_info < (3,):
+            command_line_conf = [
+                a.decode('utf-8', 'replace') for a in command_line_conf]
+
         if '--ignore-config' in command_line_conf:
             system_conf = []
             user_conf = []

From 48c971e07336f1e5d33a760e454c52fac83392d5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Mar 2015 16:39:53 +0100
Subject: [PATCH 0072/1515] release 2015.03.24

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 72b365305..baf7b3880 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -231,6 +231,7 @@
  - **Letv**
  - **LetvPlaylist**
  - **LetvTv**
+ - **Libsyn**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
@@ -310,6 +311,7 @@
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
  - **NRK**
+ - **NRKPlaylist**
  - **NRKTV**
  - **ntv.ru**
  - **Nuvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 51b4260aa..039ceadf2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.18'
+__version__ = '2015.03.24'

From 17941321ab0b3f9548d1f65e3f9d69e8cd01c0a3 Mon Sep 17 00:00:00 2001
From: testbonn <andreas.beer@gmail.com>
Date: Wed, 25 Mar 2015 11:02:55 +0100
Subject: [PATCH 0073/1515] Clean up of --help output

For consistency and readability
---
 youtube_dl/options.py | 196 +++++++++++++++++++++---------------------
 1 file changed, 98 insertions(+), 98 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 35c7e5fb3..68193a271 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -120,19 +120,19 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '-h', '--help',
         action='help',
-        help='print this help text and exit')
+        help='Print this help text and exit')
     general.add_option(
         '-v', '--version',
         action='version',
-        help='print program version and exit')
+        help='Print program version and exit')
     general.add_option(
         '-U', '--update',
         action='store_true', dest='update_self',
-        help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
+        help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
     general.add_option(
         '-i', '--ignore-errors',
         action='store_true', dest='ignoreerrors', default=False,
-        help='continue on download errors, for example to skip unavailable videos in a playlist')
+        help='Continue on download errors, for example to skip unavailable videos in a playlist')
     general.add_option(
         '--abort-on-error',
         action='store_false', dest='ignoreerrors',
@@ -140,7 +140,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--dump-user-agent',
         action='store_true', dest='dump_user_agent', default=False,
-        help='display the current browser identification')
+        help='Display the current browser identification')
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
@@ -152,7 +152,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
-        help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
+        help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
     general.add_option(
         '--ignore-config',
         action='store_true',
@@ -169,7 +169,7 @@ def parseOpts(overrideArguments=None):
         '--no-color', '--no-colors',
         action='store_true', dest='no_color',
         default=False,
-        help='Do not emit color codes in output.')
+        help='Do not emit color codes in output')
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(
@@ -206,23 +206,23 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--playlist-start',
         dest='playliststart', metavar='NUMBER', default=1, type=int,
-        help='playlist video to start at (default is %default)')
+        help='Playlist video to start at (default is %default)')
     selection.add_option(
         '--playlist-end',
         dest='playlistend', metavar='NUMBER', default=None, type=int,
-        help='playlist video to end at (default is last)')
+        help='Playlist video to end at (default is last)')
     selection.add_option(
         '--playlist-items',
         dest='playlist_items', metavar='ITEM_SPEC', default=None,
-        help='playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
+        help='Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',
-        help='download only matching titles (regex or caseless sub-string)')
+        help='Download only matching titles (regex or caseless sub-string)')
     selection.add_option(
         '--reject-title',
         dest='rejecttitle', metavar='REGEX',
-        help='skip download for matching titles (regex or caseless sub-string)')
+        help='Skip download for matching titles (regex or caseless sub-string)')
     selection.add_option(
         '--max-downloads',
         dest='max_downloads', metavar='NUMBER', type=int, default=None,
@@ -238,19 +238,19 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--date',
         metavar='DATE', dest='date', default=None,
-        help='download only videos uploaded in this date')
+        help='Download only videos uploaded in this date')
     selection.add_option(
         '--datebefore',
         metavar='DATE', dest='datebefore', default=None,
-        help='download only videos uploaded on or before this date (i.e. inclusive)')
+        help='Download only videos uploaded on or before this date (i.e. inclusive)')
     selection.add_option(
         '--dateafter',
         metavar='DATE', dest='dateafter', default=None,
-        help='download only videos uploaded on or after this date (i.e. inclusive)')
+        help='Download only videos uploaded on or after this date (i.e. inclusive)')
     selection.add_option(
         '--min-views',
         metavar='COUNT', dest='min_views', default=None, type=int,
-        help='Do not download any videos with less than COUNT views',)
+        help='Do not download any videos with less than COUNT views')
     selection.add_option(
         '--max-views',
         metavar='COUNT', dest='max_views', default=None, type=int,
@@ -259,7 +259,7 @@ def parseOpts(overrideArguments=None):
         '--match-filter',
         metavar='FILTER', dest='match_filter', default=None,
         help=(
-            '(Experimental) Generic video filter. '
+            'Generic video filter (experimental). '
             'Specify any key (see help for -o for a list of available keys) to'
             ' match if the key is present, '
             '!key to check if the key is not present,'
@@ -277,15 +277,15 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
-        help='If the URL refers to a video and a playlist, download only the video.')
+        help='Download only the video, if the URL refers to a video and a playlist.')
     selection.add_option(
         '--yes-playlist',
         action='store_false', dest='noplaylist', default=False,
-        help='If the URL refers to a video and a playlist, download the playlist.')
+        help='Download the playlist, if the URL refers to a video and a playlist.')
     selection.add_option(
         '--age-limit',
         metavar='YEARS', dest='age_limit', default=None, type=int,
-        help='download only videos suitable for the given age')
+        help='Download only videos suitable for the given age')
     selection.add_option(
         '--download-archive', metavar='FILE',
         dest='download_archive',
@@ -299,30 +299,30 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '-u', '--username',
         dest='username', metavar='USERNAME',
-        help='login with this account ID')
+        help='Login with this account ID')
     authentication.add_option(
         '-p', '--password',
         dest='password', metavar='PASSWORD',
-        help='account password. If this option is left out, youtube-dl will ask interactively.')
+        help='Account password. If this option is left out, youtube-dl will ask interactively.')
     authentication.add_option(
         '-2', '--twofactor',
         dest='twofactor', metavar='TWOFACTOR',
-        help='two-factor auth code')
+        help='Two-factor auth code')
     authentication.add_option(
         '-n', '--netrc',
         action='store_true', dest='usenetrc', default=False,
-        help='use .netrc authentication data')
+        help='Use .netrc authentication data')
     authentication.add_option(
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
-        help='video password (vimeo, smotri)')
+        help='Video password (vimeo, smotri)')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
         help=(
-            'video format code, specify the order of preference using'
+            'Video format code, specify the order of preference using'
             ' slashes, as in -f 22/17/18 . '
             ' Instead of format codes, you can select by extension for the '
             'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
@@ -350,19 +350,19 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',
-        help='download all available video formats')
+        help='Download all available video formats')
     video_format.add_option(
         '--prefer-free-formats',
         action='store_true', dest='prefer_free_formats', default=False,
-        help='prefer free video formats unless a specific one is requested')
+        help='Prefer free video formats unless a specific one is requested')
     video_format.add_option(
         '--max-quality',
         action='store', dest='format_limit', metavar='FORMAT',
-        help='highest quality format to download')
+        help='Specify highest quality format to download')
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
-        help='list all available formats')
+        help='List all available formats')
     video_format.add_option(
         '--youtube-include-dash-manifest',
         action='store_true', dest='youtube_include_dash_manifest', default=True,
@@ -382,46 +382,46 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--write-sub', '--write-srt',
         action='store_true', dest='writesubtitles', default=False,
-        help='write subtitle file')
+        help='Write subtitle file')
     subtitles.add_option(
         '--write-auto-sub', '--write-automatic-sub',
         action='store_true', dest='writeautomaticsub', default=False,
-        help='write automatic subtitle file (youtube only)')
+        help='Write automatic subtitle file (YouTube only)')
     subtitles.add_option(
         '--all-subs',
         action='store_true', dest='allsubtitles', default=False,
-        help='downloads all the available subtitles of the video')
+        help='Download all the available subtitles of the video')
     subtitles.add_option(
         '--list-subs',
         action='store_true', dest='listsubtitles', default=False,
-        help='lists all available subtitles for the video')
+        help='List all available subtitles for the video')
     subtitles.add_option(
         '--sub-format',
         action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
-        help='subtitle format, accepts formats preference, for example: "ass/srt/best"')
+        help='Specify subtitle format preference, for example: "srt" or "ass/srt/best"')
     subtitles.add_option(
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
         default=[], callback=_comma_separated_values_options_callback,
-        help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
+        help='Languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 
     downloader = optparse.OptionGroup(parser, 'Download Options')
     downloader.add_option(
         '-r', '--rate-limit',
         dest='ratelimit', metavar='LIMIT',
-        help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+        help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
     downloader.add_option(
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
-        help='number of retries (default is %default), or "infinite".')
+        help='Number of retries (default is %default), or "infinite".')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',
-        help='size of download buffer (e.g. 1024 or 16K) (default is %default)')
+        help='Size of download buffer (e.g. 1024 or 16K) (default is %default)')
     downloader.add_option(
         '--no-resize-buffer',
         action='store_true', dest='noresizebuffer', default=False,
-        help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
+        help='Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
     downloader.add_option(
         '--test',
         action='store_true', dest='test', default=False,
@@ -433,11 +433,11 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--xattr-set-filesize',
         dest='xattr_set_filesize', action='store_true',
-        help='(experimental) set file xattribute ytdl.filesize with expected filesize')
+        help='Set file xattribute ytdl.filesize with expected filesize (experimental)')
     downloader.add_option(
         '--hls-prefer-native',
         dest='hls_prefer_native', action='store_true',
-        help='(experimental) Use the native HLS downloader instead of ffmpeg.')
+        help='Use the native HLS downloader instead of ffmpeg (experimental)')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',
@@ -446,7 +446,7 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--external-downloader-args',
         dest='external_downloader_args', metavar='ARGS',
-        help='Give these arguments to the external downloader.')
+        help='Give these arguments to the external downloader')
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(
@@ -456,7 +456,7 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--no-check-certificate',
         action='store_true', dest='no_check_certificate', default=False,
-        help='Suppress HTTPS certificate validation.')
+        help='Suppress HTTPS certificate validation')
     workarounds.add_option(
         '--prefer-insecure',
         '--prefer-unsecure', action='store_true', dest='prefer_insecure',
@@ -464,16 +464,16 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--user-agent',
         metavar='UA', dest='user_agent',
-        help='specify a custom user agent')
+        help='Specify a custom user agent')
     workarounds.add_option(
         '--referer',
         metavar='URL', dest='referer', default=None,
-        help='specify a custom referer, use if the video access is restricted to one domain',
+        help='Specify a custom referer, use if the video access is restricted to one domain',
     )
     workarounds.add_option(
         '--add-header',
         metavar='FIELD:VALUE', dest='headers', action='append',
-        help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
+        help='Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
     )
     workarounds.add_option(
         '--bidi-workaround',
@@ -488,7 +488,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-q', '--quiet',
         action='store_true', dest='quiet', default=False,
-        help='activates quiet mode')
+        help='Activate quiet mode')
     verbosity.add_option(
         '--no-warnings',
         dest='no_warnings', action='store_true', default=False,
@@ -496,51 +496,51 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-s', '--simulate',
         action='store_true', dest='simulate', default=False,
-        help='do not download the video and do not write anything to disk',)
+        help='Do not download the video and do not write anything to disk')
     verbosity.add_option(
         '--skip-download',
         action='store_true', dest='skip_download', default=False,
-        help='do not download the video',)
+        help='Do not download the video')
     verbosity.add_option(
         '-g', '--get-url',
         action='store_true', dest='geturl', default=False,
-        help='simulate, quiet but print URL')
+        help='Simulate, quiet but print URL')
     verbosity.add_option(
         '-e', '--get-title',
         action='store_true', dest='gettitle', default=False,
-        help='simulate, quiet but print title')
+        help='Simulate, quiet but print title')
     verbosity.add_option(
         '--get-id',
         action='store_true', dest='getid', default=False,
-        help='simulate, quiet but print id')
+        help='Simulate, quiet but print id')
     verbosity.add_option(
         '--get-thumbnail',
         action='store_true', dest='getthumbnail', default=False,
-        help='simulate, quiet but print thumbnail URL')
+        help='Simulate, quiet but print thumbnail URL')
     verbosity.add_option(
         '--get-description',
         action='store_true', dest='getdescription', default=False,
-        help='simulate, quiet but print video description')
+        help='Simulate, quiet but print video description')
     verbosity.add_option(
         '--get-duration',
         action='store_true', dest='getduration', default=False,
-        help='simulate, quiet but print video length')
+        help='Simulate, quiet but print video length')
     verbosity.add_option(
         '--get-filename',
         action='store_true', dest='getfilename', default=False,
-        help='simulate, quiet but print output filename')
+        help='Simulate, quiet but print output filename')
     verbosity.add_option(
         '--get-format',
         action='store_true', dest='getformat', default=False,
-        help='simulate, quiet but print output format')
+        help='Simulate, quiet but print output format')
     verbosity.add_option(
         '-j', '--dump-json',
         action='store_true', dest='dumpjson', default=False,
-        help='simulate, quiet but print JSON information. See --output for a description of available keys.')
+        help='Simulate, quiet but print JSON information. See --output for a description of available keys.')
     verbosity.add_option(
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,
-        help='simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
+        help='Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
     verbosity.add_option(
         '--print-json',
         action='store_true', dest='print_json', default=False,
@@ -549,23 +549,23 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '--newline',
         action='store_true', dest='progress_with_newline', default=False,
-        help='output progress bar as new lines')
+        help='Output progress bar as new lines')
     verbosity.add_option(
         '--no-progress',
         action='store_true', dest='noprogress', default=False,
-        help='do not print progress bar')
+        help='Do not print progress bar')
     verbosity.add_option(
         '--console-title',
         action='store_true', dest='consoletitle', default=False,
-        help='display progress in console titlebar')
+        help='Display progress in console titlebar')
     verbosity.add_option(
         '-v', '--verbose',
         action='store_true', dest='verbose', default=False,
-        help='print various debugging information')
+        help='Print various debugging information')
     verbosity.add_option(
         '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
-        help='print downloaded pages to debug problems (very verbose)')
+        help='Print downloaded pages to debug problems (very verbose)')
     verbosity.add_option(
         '--write-pages',
         action='store_true', dest='write_pages', default=False,
@@ -581,31 +581,31 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-C', '--call-home',
         dest='call_home', action='store_true', default=False,
-        help='Contact the youtube-dl server for debugging.')
+        help='Contact the youtube-dl server for debugging')
     verbosity.add_option(
         '--no-call-home',
         dest='call_home', action='store_false', default=False,
-        help='Do NOT contact the youtube-dl server for debugging.')
+        help='Do NOT contact the youtube-dl server for debugging')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(
         '-a', '--batch-file',
         dest='batchfile', metavar='FILE',
-        help='file containing URLs to download (\'-\' for stdin)')
+        help='File containing URLs to download (\'-\' for stdin)')
     filesystem.add_option(
         '--id', default=False,
-        action='store_true', dest='useid', help='use only video ID in file name')
+        action='store_true', dest='useid', help='Use only video ID in file name')
     filesystem.add_option(
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
-        help=('output filename template. Use %(title)s to get the title, '
+        help=('Output filename template. Use %(title)s to get the title, '
               '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
               '%(autonumber)s to get an automatically incremented number, '
               '%(ext)s for the filename extension, '
               '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
-              '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
+              '%(format_id)s for the unique id of the format (like YouTube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
-              '%(extractor)s for the provider (youtube, metacafe, etc), '
+              '%(extractor)s for the provider (YouTube, metacafe, etc), '
               '%(id)s for the video id, '
               '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
               '%(playlist_index)s for the position in the playlist. '
@@ -617,7 +617,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--autonumber-size',
         dest='autonumber_size', metavar='NUMBER',
-        help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
+        help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
     filesystem.add_option(
         '--restrict-filenames',
         action='store_true', dest='restrictfilenames', default=False,
@@ -625,55 +625,55 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000')
+        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number of downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,
-        help='[deprecated] use title in file name (default)')
+        help='[deprecated] Use title in file name (default)')
     filesystem.add_option(
         '-l', '--literal', default=False,
         action='store_true', dest='usetitle',
-        help='[deprecated] alias of --title')
+        help='[deprecated] Alias of --title')
     filesystem.add_option(
         '-w', '--no-overwrites',
         action='store_true', dest='nooverwrites', default=False,
-        help='do not overwrite files')
+        help='Do not overwrite files')
     filesystem.add_option(
         '-c', '--continue',
         action='store_true', dest='continue_dl', default=True,
-        help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
+        help='Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
     filesystem.add_option(
         '--no-continue',
         action='store_false', dest='continue_dl',
-        help='do not resume partially downloaded files (restart from beginning)')
+        help='Do not resume partially downloaded files (restart from beginning)')
     filesystem.add_option(
         '--no-part',
         action='store_true', dest='nopart', default=False,
-        help='do not use .part files - write directly into output file')
+        help='Do not use .part files - write directly into output file')
     filesystem.add_option(
         '--no-mtime',
         action='store_false', dest='updatetime', default=True,
-        help='do not use the Last-modified header to set the file modification time')
+        help='Do not use the Last-modified header to set the file modification time')
     filesystem.add_option(
         '--write-description',
         action='store_true', dest='writedescription', default=False,
-        help='write video description to a .description file')
+        help='Write video description to a .description file')
     filesystem.add_option(
         '--write-info-json',
         action='store_true', dest='writeinfojson', default=False,
-        help='write video metadata to a .info.json file')
+        help='Write video metadata to a .info.json file')
     filesystem.add_option(
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
-        help='write video annotations to a .annotation file')
+        help='Write video annotations to a .annotation file')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
-        help='json file containing the video information (created with the "--write-json" option)')
+        help='Specify JSON file containing the video information (created with the "--write-json" option)')
     filesystem.add_option(
         '--cookies',
         dest='cookiefile', metavar='FILE',
-        help='file to read cookies from and dump cookie jar in')
+        help='File to read cookies from and dump cookie jar in')
     filesystem.add_option(
         '--cache-dir', dest='cachedir', default=None, metavar='DIR',
         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
@@ -689,11 +689,11 @@ def parseOpts(overrideArguments=None):
     thumbnail.add_option(
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,
-        help='write thumbnail image to disk')
+        help='Write thumbnail image to disk')
     thumbnail.add_option(
         '--write-all-thumbnails',
         action='store_true', dest='write_all_thumbnails', default=False,
-        help='write all thumbnail image formats to disk')
+        help='Write all thumbnail image formats to disk')
     thumbnail.add_option(
         '--list-thumbnails',
         action='store_true', dest='list_thumbnails', default=False,
@@ -703,14 +703,14 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '-x', '--extract-audio',
         action='store_true', dest='extractaudio', default=False,
-        help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
+        help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
     postproc.add_option(
         '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-        help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
+        help='Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
     postproc.add_option(
         '--audio-quality', metavar='QUALITY',
         dest='audioquality', default='5',
-        help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
+        help='Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
@@ -718,27 +718,27 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
-        help='keeps the video file on disk after the post-processing; the video is erased by default')
+        help='Keep the video file on disk after the post-processing; the video is erased by default')
     postproc.add_option(
         '--no-post-overwrites',
         action='store_true', dest='nopostoverwrites', default=False,
-        help='do not overwrite post-processed files; the post-processed files are overwritten by default')
+        help='Do not overwrite post-processed files; the post-processed files are overwritten by default')
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,
-        help='embed subtitles in the video (only for mp4 videos)')
+        help='Embed subtitles in the video (only for mp4 videos)')
     postproc.add_option(
         '--embed-thumbnail',
         action='store_true', dest='embedthumbnail', default=False,
-        help='embed thumbnail in the audio as cover art')
+        help='Embed thumbnail in the audio as cover art')
     postproc.add_option(
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
-        help='write metadata to the video file')
+        help='Write metadata to the video file')
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',
-        help='parse additional metadata like song title / artist from the video title. '
+        help='Parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
              'the parsed parameters replace existing values. '
              'Additional templates: %(album), %(artist). '
@@ -747,7 +747,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
-        help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+        help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
     postproc.add_option(
         '--fixup',
         metavar='POLICY', dest='fixup', default='detect_or_warn',

From 2676caf344c1e22f5b01588716471edeeebf0ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 20:08:35 +0600
Subject: [PATCH 0074/1515] [redtube] Capture and output removed video message
 (#5281)

---
 youtube_dl/extractor/redtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 846b76c81..2e3db6887 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class RedTubeIE(InfoExtractor):
@@ -19,6 +20,9 @@ class RedTubeIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if any(s in webpage for s in ['video-deleted-info', '>This video has been removed']):
+            raise ExtractorError('Video %s has been removed' % video_id, expected=True)
+
         video_url = self._html_search_regex(
             r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
         video_title = self._html_search_regex(

From 838b93405bc31654b07e120795115aec6e883855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 20:09:01 +0600
Subject: [PATCH 0075/1515] [redtube] Fix test

---
 youtube_dl/extractor/redtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 2e3db6887..d6054d717 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -8,11 +8,12 @@ class RedTubeIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.redtube.com/66418',
+        'md5': '7b8c22b5e7098a3e1c09709df1126d2d',
         'info_dict': {
             'id': '66418',
             'ext': 'mp4',
-            "title": "Sucked on a toilet",
-            "age_limit": 18,
+            'title': 'Sucked on a toilet',
+            'age_limit': 18,
         }
     }
 

From 5c19d18cbfb6a0271a9d0b7ff516e1e2e0eeccfb Mon Sep 17 00:00:00 2001
From: Amish Bhadeshia <amishb@gmail.com>
Date: Mon, 23 Mar 2015 13:02:44 +0000
Subject: [PATCH 0076/1515] [22Tracks] Add new extractor

Conflicts:
	youtube_dl/extractor/__init__.py
---
 youtube_dl/extractor/twentytwotracks.py | 111 ++++++++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 youtube_dl/extractor/twentytwotracks.py

diff --git a/youtube_dl/extractor/twentytwotracks.py b/youtube_dl/extractor/twentytwotracks.py
new file mode 100644
index 000000000..b655a7503
--- /dev/null
+++ b/youtube_dl/extractor/twentytwotracks.py
@@ -0,0 +1,111 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+# 22Tracks regularly replace the audio tracks that can be streamed on their
+# site. The tracks usually expire after 1 months, so we can't add tests.
+
+
+class TwentyTwoTracksIE(InfoExtractor):
+    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/(\d+)'
+    IE_NAME = 'TwentyTwoTracks:Tracks'
+
+    def _extract_info(self, city, genre, track=''):
+        self._base_url = "http://22tracks.com/api/"
+
+        if track == '':
+            itemid = genre
+        else:
+            itemid = track
+
+        cities = self._download_json(
+            self._base_url + 'cities', itemid,
+            'Downloading city info', 'Cannot download city info')
+        city_id = [x['id'] for x in cities if x['slug'] == city]
+
+        genres = self._download_json(
+            self._base_url + 'genres/' + str(city_id[0]), itemid,
+            'Downloading genre info', 'Cannot download genre info')
+        genre_id = [x['id'] for x in genres if x['slug'] == genre]
+
+        tracks = self._download_json(
+            self._base_url + 'tracks/' + str(genre_id[0]),
+            itemid, 'Downloading track info', 'Cannot download track info')
+
+        if track == '':
+            return [[x['title'] for x in genres if x['slug'] == genre][0],
+                    tracks]
+        else:
+            return [x for x in tracks if x['id'] == itemid][0]
+
+    def _get_token(self, filename, track_id):
+        token = self._download_json(
+            'http://22tracks.com/token.php?desktop=true&u=%2F128%2f{0}'.format(
+                filename), track_id, 'Finding download link...')
+
+        down_url = 'http://audio.22tracks.com{0}?st={1}&e={2}'.format(
+            token['filename'],
+            token['st'],
+            token['e'])
+
+        return down_url
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        city_id = mobj.group(1)
+        genre_id = mobj.group(2)
+        track_id = mobj.group(3)
+
+        self.to_screen(':: Track ID found! - Downloading single track')
+
+        track_info = self._extract_info(city_id, genre_id, track_id)
+
+        download_url = self._get_token(track_info['filename'], track_id)
+        title = '{0}-{1}'.format(
+            track_info['artist'].strip(), track_info['title'].strip())
+
+        return {
+            'id': track_id,
+            'url': download_url,
+            'ext': 'mp3',
+            'title': title,
+            'duration': track_info['duration']
+        }
+
+
+class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
+    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/?'
+    IE_NAME = 'TwentyTwoTracks:Genre'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        city_id = mobj.group(1)
+        genre_id = mobj.group(2)
+
+        self.to_screen(':: Track ID not found! - Downloading entire genre')
+
+        playlist_info = self._extract_info(city_id, genre_id)
+
+        entries = []
+        for track in playlist_info[1]:
+            title = '{0}-{1}'.format(
+                track['artist'].strip(), track['title'].strip())
+            entries.append({
+                'id': track['id'],
+                'url': self._get_token(track['filename'], track['id']),
+                'ext': 'mp3',
+                'title': title
+            })
+
+        self.to_screen(':: Links found - Downloading Playlist')
+
+        return {
+            '_type': 'playlist',
+            'id': genre_id,
+            'title': playlist_info[0],
+            'entries': entries
+        }

From 8f76df7f3791bb085199eb90c3b4125931a44e55 Mon Sep 17 00:00:00 2001
From: Amish Bhadeshia <amishb@gmail.com>
Date: Tue, 24 Mar 2015 18:34:33 +0000
Subject: [PATCH 0077/1515] Updated init to add 22tracks

---
 youtube_dl/extractor/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d73826d44..b0df52afd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -526,6 +526,10 @@ from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
+from .twentytwotracks import (
+    TwentyTwoTracksIE,
+    TwentyTwoTracksGenreIE
+)
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,

From ae67d082fe0469109033714360b2b1ae92d7d86e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 22:26:02 +0600
Subject: [PATCH 0078/1515] [22tracks] Improve and simplify

---
 youtube_dl/extractor/twentytwotracks.py | 121 ++++++++++--------------
 1 file changed, 48 insertions(+), 73 deletions(-)

diff --git a/youtube_dl/extractor/twentytwotracks.py b/youtube_dl/extractor/twentytwotracks.py
index b655a7503..d6c0ab184 100644
--- a/youtube_dl/extractor/twentytwotracks.py
+++ b/youtube_dl/extractor/twentytwotracks.py
@@ -3,109 +3,84 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 # 22Tracks regularly replace the audio tracks that can be streamed on their
 # site. The tracks usually expire after 1 months, so we can't add tests.
 
 
 class TwentyTwoTracksIE(InfoExtractor):
-    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/(\d+)'
-    IE_NAME = 'TwentyTwoTracks:Tracks'
+    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/(?P<id>\d+)'
+    IE_NAME = '22tracks:track'
 
-    def _extract_info(self, city, genre, track=''):
-        self._base_url = "http://22tracks.com/api/"
+    _API_BASE = 'http://22tracks.com/api'
 
-        if track == '':
-            itemid = genre
-        else:
-            itemid = track
+    def _extract_info(self, city, genre_name, track_id=None):
+        item_id = track_id if track_id else genre_name
 
         cities = self._download_json(
-            self._base_url + 'cities', itemid,
-            'Downloading city info', 'Cannot download city info')
-        city_id = [x['id'] for x in cities if x['slug'] == city]
+            '%s/cities' % self._API_BASE, item_id,
+            'Downloading cities info',
+            'Unable to download cities info')
+        city_id = [x['id'] for x in cities if x['slug'] == city][0]
 
         genres = self._download_json(
-            self._base_url + 'genres/' + str(city_id[0]), itemid,
-            'Downloading genre info', 'Cannot download genre info')
-        genre_id = [x['id'] for x in genres if x['slug'] == genre]
+            '%s/genres/%s' % (self._API_BASE, city_id), item_id,
+            'Downloading %s genres info' % city,
+            'Unable to download %s genres info' % city)
+        genre = [x for x in genres if x['slug'] == genre_name][0]
+        genre_id = genre['id']
 
         tracks = self._download_json(
-            self._base_url + 'tracks/' + str(genre_id[0]),
-            itemid, 'Downloading track info', 'Cannot download track info')
+            '%s/tracks/%s' % (self._API_BASE, genre_id), item_id,
+            'Downloading %s genre tracks info' % genre_name,
+            'Unable to download track info')
 
-        if track == '':
-            return [[x['title'] for x in genres if x['slug'] == genre][0],
-                    tracks]
-        else:
-            return [x for x in tracks if x['id'] == itemid][0]
+        return [x for x in tracks if x['id'] == item_id][0] if track_id else [genre['title'], tracks]
 
-    def _get_token(self, filename, track_id):
+    def _get_track_url(self, filename, track_id):
         token = self._download_json(
-            'http://22tracks.com/token.php?desktop=true&u=%2F128%2f{0}'.format(
-                filename), track_id, 'Finding download link...')
-
-        down_url = 'http://audio.22tracks.com{0}?st={1}&e={2}'.format(
-            token['filename'],
-            token['st'],
-            token['e'])
-
-        return down_url
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        city_id = mobj.group(1)
-        genre_id = mobj.group(2)
-        track_id = mobj.group(3)
-
-        self.to_screen(':: Track ID found! - Downloading single track')
-
-        track_info = self._extract_info(city_id, genre_id, track_id)
-
-        download_url = self._get_token(track_info['filename'], track_id)
-        title = '{0}-{1}'.format(
-            track_info['artist'].strip(), track_info['title'].strip())
+            'http://22tracks.com/token.php?desktop=true&u=/128/%s' % filename,
+            track_id, 'Downloading token', 'Unable to download token')
+        return 'http://audio.22tracks.com%s?st=%s&e=%d' % (token['filename'], token['st'], token['e'])
 
+    def _extract_track_info(self, track_info, track_id):
+        download_url = self._get_track_url(track_info['filename'], track_id)
+        title = '%s - %s' % (track_info['artist'].strip(), track_info['title'].strip())
         return {
             'id': track_id,
             'url': download_url,
             'ext': 'mp3',
             'title': title,
-            'duration': track_info['duration']
+            'duration': int_or_none(track_info.get('duration')),
+            'timestamp': int_or_none(track_info.get('published_at') or track_info.get('created'))
         }
 
-
-class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
-    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/?'
-    IE_NAME = 'TwentyTwoTracks:Genre'
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
 
-        city_id = mobj.group(1)
-        genre_id = mobj.group(2)
+        city = mobj.group('city')
+        genre = mobj.group('genre')
+        track_id = mobj.group('id')
 
-        self.to_screen(':: Track ID not found! - Downloading entire genre')
+        track_info = self._extract_info(city, genre, track_id)
+        return self._extract_track_info(track_info, track_id)
 
-        playlist_info = self._extract_info(city_id, genre_id)
 
-        entries = []
-        for track in playlist_info[1]:
-            title = '{0}-{1}'.format(
-                track['artist'].strip(), track['title'].strip())
-            entries.append({
-                'id': track['id'],
-                'url': self._get_token(track['filename'], track['id']),
-                'ext': 'mp3',
-                'title': title
-            })
+class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
+    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/?$'
+    IE_NAME = '22tracks:genre'
 
-        self.to_screen(':: Links found - Downloading Playlist')
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
 
-        return {
-            '_type': 'playlist',
-            'id': genre_id,
-            'title': playlist_info[0],
-            'entries': entries
-        }
+        city = mobj.group('city')
+        genre = mobj.group('genre')
+
+        genre_title, tracks = self._extract_info(city, genre)
+
+        entries = [
+            self._extract_track_info(track_info, track_info['id'])
+            for track_info in tracks]
+
+        return self.playlist_result(entries, genre, genre_title)

From 094ce39c45422d4675228337f115a110c718d68a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 22:27:20 +0600
Subject: [PATCH 0079/1515] Credit @amishb for 22tracks (#5276)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 512469f4c..59f1b5f21 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -117,3 +117,4 @@ Alexander Mamay
 Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
+Amish Bhadeshia

From db40364b8767839cc1abf5ece53511167d70f792 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 18:17:21 +0430
Subject: [PATCH 0080/1515] [Varzesh3] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/varzesh3.py | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/varzesh3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b0df52afd..3011b784d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -548,6 +548,7 @@ from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE
+from .varzesh3 import Varzesh3IE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE
 from .veoh import VeohIE
diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
new file mode 100644
index 000000000..85dcd8734
--- /dev/null
+++ b/youtube_dl/extractor/varzesh3.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+)
+import re
+
+
+class Varzesh3IE(InfoExtractor):
+    _VALID_URL = r'(?P<url>(https?://(?:www\.)?video\.varzesh3\.com)/(?P<id>.+))'
+    _TESTS = [
+    {
+        'url': 'http://video.varzesh3.com/video/%D8%AF%D8%A7%D9%86%D9%85%D8%A7%D8%B1%DA%A9-3-2-%D8%A2%D9%85%D8%B1%DB%8C%DA%A9%D8%A7/',
+        'md5': 'c4b850780df9374b078463c5cf6b3318',
+        'info_dict': {
+            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/international/friendly/usa_2_3_danmark.mp4',
+            'id': '76464',
+            'ext': 'mp4',
+            'title': u'دانمارک 	۳-۲ 	آمریکا (گلهای بازی)',
+            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/250315_usa_danmark_site.jpg',
+            'description': u'بازی دوستانه ۲۰۱۵',
+
+        }
+    },
+    {
+        'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
+        'md5': '2a933874cb7dce4366075281eb49e855',
+        'info_dict': {
+            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/video/namayeshi/saves_week26.mp4',
+            'id': '76337',
+            'ext': 'mp4',
+            'title': u'۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
+            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
+            'description': u'فصل ۲۰۱۵-۲۰۱۴',
+        }
+    },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        if not 'shortlink' in webpage:
+            raise ExtractorError('URL has no videos or there is a problem.')
+
+        title = self._html_search_regex(r'meta[^>]+property="og:title"[^>]+content="([^"]+)"', webpage, 'title')
+        video_link = self._html_search_regex(r'source[^>]+src="([^"]+)"', webpage, 'video_link')
+        vid_id = self._html_search_regex(r"link[^>]+rel='canonical'[^>]+href='\/\?p=([^']+)'\/>", webpage, 'vid_id')
+        try:
+            description = self._html_search_regex(r'<div class="matn">(.*?)</div>', webpage, 'description', flags=re.DOTALL)
+        except:
+            description = title
+        thumbnail = self._html_search_regex(r'link[^>]+rel="image_src"[^>]+href="([^"]+)"', webpage, 'thumbnail')
+
+        return {
+            'url': video_link,
+            'id': vid_id,
+            'title': title,
+            'ext': video_link.split(".")[-1],
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From fbfcc2972b3b24bda092eaed92b81113154c4327 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 16:13:53 +0200
Subject: [PATCH 0081/1515] [teamcoco] Fix extraction

---
 youtube_dl/extractor/teamcoco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 7cb06f351..a46a7ecba 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -54,7 +54,7 @@ class TeamcocoIE(InfoExtractor):
             embed_url, video_id, 'Downloading embed page')
 
         player_data = self._parse_json(self._search_regex(
-            r'Y\.Ginger\.Module\.Player\((\{.*?\})\);', embed, 'player data'), video_id)
+            r'Y\.Ginger\.Module\.Player(?:;var\s*player\s*=\s*new\s*m)?\((\{.*?\})\);', embed, 'player data'), video_id)
         data = self._parse_json(
             base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id)
 

From 91757b0f373ec3201f95066eeb0e09ebdcc1a067 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 17:15:27 +0200
Subject: [PATCH 0082/1515] [utils] Escape all HTML entities written in
 hexadecimal form

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a8ab87685..abaf1ab73 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -200,6 +200,8 @@ class TestUtil(unittest.TestCase):
 
     def test_unescape_html(self):
         self.assertEqual(unescapeHTML('%20;'), '%20;')
+        self.assertEqual(unescapeHTML('&#x2F;'), '/')
+        self.assertEqual(unescapeHTML('&#47;'), '/')
         self.assertEqual(
             unescapeHTML('&eacute;'), 'é')
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 472d4df41..245d623d8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -348,7 +348,7 @@ def _htmlentity_transform(entity):
     if entity in compat_html_entities.name2codepoint:
         return compat_chr(compat_html_entities.name2codepoint[entity])
 
-    mobj = re.match(r'#(x?[0-9]+)', entity)
+    mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
         if numstr.startswith('x'):

From a7fce980adeb12afd94e4414cc6a6b37021f6aa4 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 19:47:34 +0430
Subject: [PATCH 0083/1515] removed one of tests that made problem with testing
 server

---
 youtube_dl/extractor/varzesh3.py | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index 85dcd8734..cb3fdacff 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -9,21 +9,7 @@ import re
 
 class Varzesh3IE(InfoExtractor):
     _VALID_URL = r'(?P<url>(https?://(?:www\.)?video\.varzesh3\.com)/(?P<id>.+))'
-    _TESTS = [
-    {
-        'url': 'http://video.varzesh3.com/video/%D8%AF%D8%A7%D9%86%D9%85%D8%A7%D8%B1%DA%A9-3-2-%D8%A2%D9%85%D8%B1%DB%8C%DA%A9%D8%A7/',
-        'md5': 'c4b850780df9374b078463c5cf6b3318',
-        'info_dict': {
-            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/international/friendly/usa_2_3_danmark.mp4',
-            'id': '76464',
-            'ext': 'mp4',
-            'title': u'دانمارک 	۳-۲ 	آمریکا (گلهای بازی)',
-            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/250315_usa_danmark_site.jpg',
-            'description': u'بازی دوستانه ۲۰۱۵',
-
-        }
-    },
-    {
+    _TEST ={
         'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
         'md5': '2a933874cb7dce4366075281eb49e855',
         'info_dict': {
@@ -34,8 +20,7 @@ class Varzesh3IE(InfoExtractor):
             'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
             'description': u'فصل ۲۰۱۵-۲۰۱۴',
         }
-    },
-    ]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 8896b614a9257ba33346f49a135371d460bfb311 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 20:06:50 +0430
Subject: [PATCH 0084/1515] removing unicode literal because it is imported :))

---
 youtube_dl/extractor/varzesh3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index cb3fdacff..62bcb9118 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -16,7 +16,7 @@ class Varzesh3IE(InfoExtractor):
             'url': 'http://dl1.video.varzesh3.com/video/clip94/1/video/namayeshi/saves_week26.mp4',
             'id': '76337',
             'ext': 'mp4',
-            'title': u'۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
+            'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
             'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
             'description': u'فصل ۲۰۱۵-۲۰۱۴',
         }

From 448830ce7b180250a9f2cbafc1625a20584e59d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Mar 2015 21:41:09 +0600
Subject: [PATCH 0085/1515] [youtube:watchlater] Extract watchlater as playlist
 (Closes #5280)

---
 youtube_dl/extractor/youtube.py | 52 ++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 27c8c4453..e0f9228a4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1263,27 +1263,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
 
         return self.playlist_result(url_results, playlist_id, title)
 
-    def _real_extract(self, url):
-        # Extract playlist id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-        playlist_id = mobj.group(1) or mobj.group(2)
-
-        # Check if it's a video-specific URL
-        query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        if 'v' in query_dict:
-            video_id = query_dict['v'][0]
-            if self._downloader.params.get('noplaylist'):
-                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
-                return self.url_result(video_id, 'Youtube', video_id=video_id)
-            else:
-                self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
-
-        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
-            # Mixes require a custom extraction process
-            return self._extract_mix(playlist_id)
-
+    def _extract_playlist(self, playlist_id):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
         more_widget_html = content_html = page
@@ -1327,6 +1307,29 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         url_results = self._ids_to_results(ids)
         return self.playlist_result(url_results, playlist_id, playlist_title)
 
+    def _real_extract(self, url):
+        # Extract playlist id
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            raise ExtractorError('Invalid URL: %s' % url)
+        playlist_id = mobj.group(1) or mobj.group(2)
+
+        # Check if it's a video-specific URL
+        query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        if 'v' in query_dict:
+            video_id = query_dict['v'][0]
+            if self._downloader.params.get('noplaylist'):
+                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
+                return self.url_result(video_id, 'Youtube', video_id=video_id)
+            else:
+                self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+
+        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
+            # Mixes require a custom extraction process
+            return self._extract_mix(playlist_id)
+
+        return self._extract_playlist(playlist_id)
+
 
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
@@ -1649,13 +1652,16 @@ class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
     _PLAYLIST_TITLE = 'Youtube Recommended videos'
 
 
-class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
+class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
     _FEED_NAME = 'watch_later'
     _PLAYLIST_TITLE = 'Youtube Watch Later'
     _PERSONAL_FEED = True
 
+    def _real_extract(self, url):
+        return self._extract_playlist('WL')
+
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'

From 7e17ec8c717796ec304b205059c741e7b47474dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Mar 2015 21:42:28 +0600
Subject: [PATCH 0086/1515] [youtube] Clarify some IE_NAMEs

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e0f9228a4..eba699c3a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1646,6 +1646,7 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
 
 
 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
+    IE_NAME = 'youtube:recommended'
     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
     _FEED_NAME = 'recommended'
@@ -1653,6 +1654,7 @@ class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
 
 
 class YoutubeWatchLaterIE(YoutubePlaylistIE):
+    IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
     _FEED_NAME = 'watch_later'
@@ -1664,6 +1666,7 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
 
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+    IE_NAME = 'youtube:history'
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
     _FEED_NAME = 'history'

From 425142be601adfb4218f78c10aedbed14ad1facd Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 17:46:20 +0200
Subject: [PATCH 0087/1515] [slideshare] Fix extraction (#5279)

---
 youtube_dl/extractor/slideshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 9f79ff5c1..0b717a1e4 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -30,7 +30,7 @@ class SlideshareIE(InfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
         slideshare_obj = self._search_regex(
-            r'var\s+slideshare_object\s*=\s*({.*?});\s*var\s+user_info\s*=',
+            r'\$\.extend\(slideshare_object,\s*(\{.*?\})\);',
             webpage, 'slideshare object')
         info = json.loads(slideshare_obj)
         if info['slideshow']['type'] != 'video':

From 223b27f46cdf3cde9c9556dac19055ad5d8bff5f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 19:48:22 +0200
Subject: [PATCH 0088/1515] [vessel] Add new extractor (Closes #5275)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/vessel.py   | 127 +++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 youtube_dl/extractor/vessel.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b0df52afd..053eb32ae 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -551,6 +551,7 @@ from .ustream import UstreamIE, UstreamChannelIE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE
 from .veoh import VeohIE
+from .vessel import VesselIE
 from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import VGTVIE
diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
new file mode 100644
index 000000000..123d9470e
--- /dev/null
+++ b/youtube_dl/extractor/vessel.py
@@ -0,0 +1,127 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
+
+
+class VesselIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vessel\.com/videos/(?P<id>[0-9a-zA-Z]+)'
+    _API_URL_TEMPLATE = 'https://www.vessel.com/api/view/items/%s'
+    _LOGIN_URL = 'https://www.vessel.com/api/account/login'
+    _NETRC_MACHINE = 'vessel'
+    _TEST = {
+        'url': 'https://www.vessel.com/videos/HDN7G5UMs',
+        'md5': '455cdf8beb71c6dd797fd2f3818d05c4',
+        'info_dict': {
+            'id': 'HDN7G5UMs',
+            'ext': 'mp4',
+            'title': 'Nvidia GeForce GTX Titan X - The Best Video Card on the Market?',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150317',
+            'description': 'Did Nvidia pull out all the stops on the Titan X, or does its performance leave something to be desired?',
+            'timestamp': int,
+        },
+    }
+
+    @staticmethod
+    def make_json_request(url, data):
+        payload = json.dumps(data).encode('utf-8')
+        req = compat_urllib_request.Request(url, payload)
+        req.add_header('Content-Type', 'application/json; charset=utf-8')
+        return req
+
+    @staticmethod
+    def find_assets(data, asset_type):
+        for asset in data.get('assets', []):
+            if asset.get('type') == asset_type:
+                yield asset
+
+    def _check_access_rights(self, data):
+        access_info = data.get('__view', {})
+        if access_info.get('allow_access') == False:
+            err_code = access_info.get('error_code') or ''
+            if err_code == 'ITEM_PAID_ONLY':
+                raise ExtractorError(
+                    'This video requires subscription.', expected=True)
+            else:
+                raise ExtractorError(
+                    'Access to this content is restricted. (%s said: %s)' % (self.IE_NAME, err_code), expected=True)
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        data = {
+            'client_id': 'web',
+            'type': 'password',
+            'user_key': username,
+            'password': password,
+        }
+        login_request = VesselIE.make_json_request(self._LOGIN_URL, data)
+        self._download_webpage(login_request, None, False, 'Wrong login info')
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        data = self._parse_json(self._search_regex(
+            r'App\.bootstrapData\((.*?)\);', webpage, 'data'), video_id)
+        asset_id = data['model']['data']['id']
+
+        req = VesselIE.make_json_request(
+            self._API_URL_TEMPLATE % asset_id, {'client': 'web'})
+        data = self._download_json(req, video_id)
+
+        self._check_access_rights(data)
+
+        try:
+            video_asset = next(VesselIE.find_assets(data, 'video'))
+        except StopIteration:
+            raise ExtractorError('No video assets found')
+
+        formats = []
+        for f in video_asset.get('sources', []):
+            if f['name'] == 'hls-index':
+                formats.extend(self._extract_m3u8_formats(
+                    f['location'], video_id, ext='mp4', m3u8_id='m3u8'))
+            else:
+                formats.append({
+                    'format_id': f['name'],
+                    'tbr': f.get('bitrate'),
+                    'height': f.get('height'),
+                    'width': f.get('width'),
+                    'url': f['location'],
+                })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for im_asset in VesselIE.find_assets(data, 'image'):
+            thumbnails.append({
+                'url': im_asset['location'],
+                'width': im_asset.get('width', 0),
+                'height': im_asset.get('height', 0),
+            })
+
+        return {
+            'id': video_id,
+            'title': data['title'],
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': data.get('short_description'),
+            'duration': data.get('duration'),
+            'comment_count': data.get('comment_count'),
+            'like_count': data.get('like_count'),
+            'view_count': data.get('view_count'),
+            'timestamp': parse_iso8601(data.get('released_at')),
+        }

From 31c48098276100a3fa2529b1443687e56838f707 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Mar 2015 23:57:46 +0600
Subject: [PATCH 0089/1515] [safari] Improve and simplify

---
 youtube_dl/extractor/safari.py | 101 +++++++++++++++++++--------------
 1 file changed, 57 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 3e494b960..10251f29e 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveIE
@@ -20,16 +19,18 @@ from ..utils import (
 
 class SafariBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
-    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]+>Sign Out</a>'
-    _ACCOUNT_CREDENTIALS_HINT = ('Use --username and --password options to '
-                                 'supply credentials for safaribooksonline.com ')
-    _NETRC_MACHINE = 'safaribooksonline'
+    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
+    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to supply credentials for safaribooksonline.com'
+    _NETRC_MACHINE = 'safari'
+
+    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
+    _API_FORMAT = 'json'
 
     LOGGED_IN = False
 
     def _real_initialize(self):
         # We only need to log in once for courses or individual videos
-        if not SafariBaseIE.LOGGED_IN:
+        if not self.LOGGED_IN:
             self._login()
             SafariBaseIE.LOGGED_IN = True
 
@@ -49,7 +50,7 @@ class SafariBaseIE(InfoExtractor):
             'Downloading login form')
 
         csrf = self._html_search_regex(
-            r"<input +type='hidden' +name='csrfmiddlewaretoken' +value='([^']+)' +/>",
+            r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
             login_page, 'csrf token')
 
         login_form = {
@@ -66,8 +67,9 @@ class SafariBaseIE(InfoExtractor):
             request, None, 'Logging in as %s' % username)
 
         if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
-            raise ExtractorError('Login failed; make sure your credentials are correct and '
-                                 'try again.', expected=True)
+            raise ExtractorError(
+                'Login failed; make sure your credentials are correct and try again.',
+                expected=True)
 
         self.to_screen('Login successful')
 
@@ -75,69 +77,80 @@ class SafariBaseIE(InfoExtractor):
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'
     IE_DESC = 'safaribooksonline.com online video'
-    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/'
-                  '(?P<id>\d+)/(?P<part>part\d+)\.html')
-    _TEST = {
-        'url': ('https://www.safaribooksonline.com/library/view/'
-                'hadoop-fundamentals-livelessons/9780133392838/part00.html'),
+    _VALID_URL = r'''(?x)https?://
+                            (?:www\.)?safaribooksonline\.com/
+                                (?:
+                                    library/view/[^/]+|
+                                    api/v1/book
+                                )/
+                                (?P<course_id>\d+)/
+                                    (?:chapter(?:-content)?/)?
+                                (?P<part>part\d+)\.html
+    '''
+
+    _TESTS = [{
+        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
         'md5': '5b0c4cc1b3c1ba15dda7344085aa5592',
         'info_dict': {
-            'id': '9780133392838',
+            'id': '2842601850001',
             'ext': 'mp4',
             'title': 'Introduction',
-        }
-    }
+        },
+        'skip': 'Requires safaribooksonline account credentials',
+    }, {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        course_id = mobj.group('course_id')
         part = mobj.group('part')
 
-        webpage = self._download_webpage(url, part)
+        webpage = self._download_webpage(
+            '%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
+            part)
+
         bc_url = BrightcoveIE._extract_brightcove_url(webpage)
         if not bc_url:
             raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
 
-        return {
-            '_type': 'url',
-            'url': smuggle_url(bc_url, {'Referer': url}),
-            'ie_key': 'Brightcove'
-        }
+        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'Brightcove')
 
 
 class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
 
-    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/'
-                  '(?P<course_path>[^/]+)/(?P<id>\d+)/?$')
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>\d+)/?(?:[#?]|$)'
 
-    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
-    _API_FORMAT = 'json'
+    _TESTS = [{
+        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
+        'info_dict': {
+            'id': '9780133392838',
+            'title': 'Hadoop Fundamentals LiveLessons',
+        },
+        'playlist_count': 22,
+        'skip': 'Requires safaribooksonline account credentials',
+    }, {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        course_path = mobj.group('course_path')
-        course_id = mobj.group('id')
+        course_id = self._match_id(url)
 
-        webpage = self._download_webpage(
+        course_json = self._download_json(
             '%s/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
-            course_path, 'Downloading course JSON')
-
-        course_json = json.loads(webpage)
+            course_id, 'Downloading course JSON')
 
         if 'chapters' not in course_json:
-            raise ExtractorError('No chapters found for course %s' % course_id, expected=True)
-
-        num_parts = len(course_json['chapters'])
-        parts = ['%02d' % part for part in range(num_parts)]
+            raise ExtractorError(
+                'No chapters found for course %s' % course_id, expected=True)
 
         entries = [
-            self.url_result(
-                'https://www.safaribooksonline.com/library/view/%s/%s/part%s.html' % (course_path,
-                                                                                      course_id,
-                                                                                      part_id),
-                'Safari')
-            for part_id in parts]
+            self.url_result(chapter, 'Safari')
+            for chapter in course_json['chapters']]
 
         course_title = course_json['title']
 

From c496ec084823436fa27eda0a12887c028f572ed3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 26 Mar 2015 19:51:40 +0100
Subject: [PATCH 0090/1515] [vessel] Fix pep8 issue

---
 youtube_dl/extractor/vessel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 123d9470e..6215f0642 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -45,7 +45,7 @@ class VesselIE(InfoExtractor):
 
     def _check_access_rights(self, data):
         access_info = data.get('__view', {})
-        if access_info.get('allow_access') == False:
+        if not access_info.get('allow_access', True):
             err_code = access_info.get('error_code') or ''
             if err_code == 'ITEM_PAID_ONLY':
                 raise ExtractorError(

From 157e9e5aa5e2e461ec0e4f1e0875125a440d6d60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 26 Mar 2015 20:03:31 +0100
Subject: [PATCH 0091/1515] [youtube:watchlater] Remove unused properties and
 fix tests

---
 test/test_all_urls.py           | 2 +-
 youtube_dl/extractor/youtube.py | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 6ae168b7f..a9db42b30 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -59,7 +59,7 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
 
     def test_youtube_feeds(self):
-        self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watch_later'])
+        self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watchlater'])
         self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:subscriptions'])
         self.assertMatch('https://www.youtube.com/feed/recommended', ['youtube:recommended'])
         self.assertMatch('https://www.youtube.com/my_favorites', ['youtube:favorites'])
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eba699c3a..5488101e1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1657,9 +1657,8 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
-    _FEED_NAME = 'watch_later'
-    _PLAYLIST_TITLE = 'Youtube Watch Later'
-    _PERSONAL_FEED = True
+
+    _TESTS = []  # override PlaylistIE tests
 
     def _real_extract(self, url):
         return self._extract_playlist('WL')

From 2315fb5e5fb65271519fc018bb1df5a7269f0a6a Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 23:53:57 +0430
Subject: [PATCH 0092/1515] unicde :(

---
 youtube_dl/extractor/varzesh3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index 62bcb9118..eb49586cc 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -18,7 +18,7 @@ class Varzesh3IE(InfoExtractor):
             'ext': 'mp4',
             'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
             'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
-            'description': u'فصل ۲۰۱۵-۲۰۱۴',
+            'description': 'فصل ۲۰۱۵-۲۰۱۴',
         }
     }
 

From 998e6cdba01ff54e2009d50c3026e7af0bb1c11a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Mar 2015 03:05:08 +0600
Subject: [PATCH 0093/1515] [vimeo] Capture and output error message (#5294)

---
 youtube_dl/extractor/vimeo.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bd09652cd..28bcc89cd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -244,6 +244,16 @@ class VimeoIE(VimeoBaseInfoExtractor):
         # and latter we extract those that are Vimeo specific.
         self.report_extraction(video_id)
 
+        vimeo_config = self._search_regex(
+            r'vimeo\.config\s*=\s*({.+?});', webpage,
+            'vimeo config', default=None)
+        if vimeo_config:
+            seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
+            if seed_status.get('state') == 'failed':
+                raise ExtractorError(
+                    '%s returned error: %s' % (self.IE_NAME, seed_status['title']),
+                    expected=True)
+
         # Extract the config JSON
         try:
             try:

From af140002158c1079b1365392c6a48ea06ad23c82 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 23:24:15 +0200
Subject: [PATCH 0094/1515] [eroprofile] Add login support (#5269)

---
 youtube_dl/extractor/eroprofile.py | 52 ++++++++++++++++++++++++++++--
 1 file changed, 50 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 79e2fbd39..0cbca90b0 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -1,11 +1,17 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import ExtractorError
 
 
 class EroProfileIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<id>[^/]+)'
-    _TEST = {
+    _LOGIN_URL = 'http://www.eroprofile.com/auth/auth.php?'
+    _NETRC_MACHINE = 'eroprofile'
+    _TESTS = [{
         'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
         'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
         'info_dict': {
@@ -16,13 +22,55 @@ class EroProfileIE(InfoExtractor):
             'thumbnail': 're:https?://.*\.jpg',
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'http://www.eroprofile.com/m/videos/view/Try-It-On-Pee_cut_2-wmv-4shared-com-file-sharing-download-movie-file',
+        'md5': '1baa9602ede46ce904c431f5418d8916',
+        'info_dict': {
+            'id': '1133519',
+            'ext': 'm4v',
+            'title': 'Try It On Pee_cut_2.wmv - 4shared.com - file sharing - download movie file',
+            'thumbnail': 're:https?://.*\.jpg',
+            'age_limit': 18,
+        },
+        'skip': 'Requires login',
+    }]
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        query = compat_urllib_parse.urlencode({
+            'username': username,
+            'password': password,
+            'url': 'http://www.eroprofile.com/',
+        })
+        login_url = self._LOGIN_URL + query
+        login_page = self._download_webpage(login_url, None, False)
+
+        m = re.search(r'Your username or password was incorrect\.', login_page)
+        if m:
+            raise ExtractorError(
+                'Wrong username and/or password.', expected=True)
+
+        self.report_login()
+        redirect_url = self._search_regex(
+            r'<script[^>]+?src="([^"]+)"', login_page, 'login redirect url')
+        self._download_webpage(redirect_url, None, False)
+
+    def _real_initialize(self):
+        self._login()
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
+        m = re.search(r'You must be logged in to view this video\.', webpage)
+        if m:
+            raise ExtractorError(
+                'This video requires login. Please specify a username and password and try again.', expected=True)
+
         video_id = self._search_regex(
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)

From 70a1165b32acf253905109e9b4f245295d67af1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Mar 2015 13:02:20 +0100
Subject: [PATCH 0095/1515] Don't use bare 'except:'

They catch any exception, including KeyboardInterrupt, we don't want to catch it.
---
 devscripts/check-porn.py           |  2 +-
 youtube_dl/YoutubeDL.py            |  4 ++--
 youtube_dl/compat.py               |  2 +-
 youtube_dl/downloader/common.py    |  2 +-
 youtube_dl/extractor/youporn.py    |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 15 ++++++---------
 youtube_dl/update.py               |  4 ++--
 youtube_dl/utils.py                |  4 ++--
 8 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 6a5bd9eda..7a219ebe9 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -28,7 +28,7 @@ for test in get_testcases():
     if METHOD == 'EURISTIC':
         try:
             webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
-        except:
+        except Exception:
             print('\nFail: {0}'.format(test['name']))
             continue
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5ef5e009..640b8c99d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1701,10 +1701,10 @@ class YoutubeDL(object):
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
                 self._write_string('[debug] Git HEAD: ' + out + '\n')
-        except:
+        except Exception:
             try:
                 sys.exc_clear()
-            except:
+            except Exception:
                 pass
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b2bf149ef..973bcd320 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -389,7 +389,7 @@ else:
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             out, err = sp.communicate()
             lines, columns = map(int, out.split())
-        except:
+        except Exception:
             pass
         return _terminal_size(columns, lines)
 
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 8ed5c19a6..ca14d64bc 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -204,7 +204,7 @@ class FileDownloader(object):
             return
         try:
             os.utime(filename, (time.time(), filetime))
-        except:
+        except Exception:
             pass
         return filetime
 
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index e4c855ee0..6abe72f73 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -52,7 +52,7 @@ class YouPornIE(InfoExtractor):
             webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)
-        except:
+        except ValueError:
             raise ExtractorError('Invalid JSON')
 
         self.report_extraction(video_id)
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index b6f51cfd5..55adf9685 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import io
 import os
 import subprocess
-import sys
 import time
 
 
@@ -269,19 +268,17 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             else:
                 self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
-        except:
-            etype, e, tb = sys.exc_info()
-            if isinstance(e, AudioConversionError):
-                msg = 'audio conversion failed: ' + e.msg
-            else:
-                msg = 'error running ' + self.basename
-            raise PostProcessingError(msg)
+        except AudioConversionError as e:
+            raise PostProcessingError(
+                'audio conversion failed: ' + e.msg)
+        except Exception:
+            raise PostProcessingError('error running ' + self.basename)
 
         # Try to update the date time for extracted audio file.
         if information.get('filetime') is not None:
             try:
                 os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-            except:
+            except Exception:
                 self._downloader.report_warning('Cannot update utime of audio file')
 
         information['filepath'] = new_path
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index d8be4049f..de3169eef 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -65,7 +65,7 @@ def update_self(to_screen, verbose):
     # Check if there is a new version
     try:
         newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
-    except:
+    except Exception:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t find the current version. Please try again later.')
@@ -78,7 +78,7 @@ def update_self(to_screen, verbose):
     try:
         versions_info = opener.open(JSON_URL).read().decode('utf-8')
         versions_info = json.loads(versions_info)
-    except:
+    except Exception:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t obtain versions info. Please try again later.')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 245d623d8..90e0ed9ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -75,7 +75,7 @@ def preferredencoding():
     try:
         pref = locale.getpreferredencoding()
         'TEST'.encode(pref)
-    except:
+    except Exception:
         pref = 'UTF-8'
 
     return pref
@@ -127,7 +127,7 @@ def write_json_file(obj, fn):
             except OSError:
                 pass
         os.rename(tf.name, fn)
-    except:
+    except Exception:
         try:
             os.remove(tf.name)
         except OSError:

From 8e678af4ba873c348ec2634ab22f84c61546c093 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Mar 2015 14:21:53 +0100
Subject: [PATCH 0096/1515] Makefile: fix 'find' command

It worked with the GNU version, but not with the BSD version.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c6c76274f..fdb1abb60 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bas
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
-	find -name "*.pyc" -delete
+	find . -name "*.pyc" -delete
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From c59e701e35dedb40da9d1e88c051141e63ded550 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 08:11:32 +0100
Subject: [PATCH 0097/1515] Default to continuedl=True

We already do this in the CLI interface, so it should be just fine.
---
 youtube_dl/downloader/common.py | 2 +-
 youtube_dl/downloader/http.py   | 2 +-
 youtube_dl/downloader/rtmp.py   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index ca14d64bc..a0fc5ead0 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -318,7 +318,7 @@ class FileDownloader(object):
         )
 
         continuedl_and_exists = (
-            self.params.get('continuedl', False) and
+            self.params.get('continuedl', True) and
             os.path.isfile(encodeFilename(filename)) and
             not self.params.get('nopart', False)
         )
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 4047d7167..d136bebd1 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -49,7 +49,7 @@ class HttpFD(FileDownloader):
 
         open_mode = 'wb'
         if resume_len != 0:
-            if self.params.get('continuedl', False):
+            if self.params.get('continuedl', True):
                 self.report_resuming_byte(resume_len)
                 request.add_header('Range', 'bytes=%d-' % resume_len)
                 open_mode = 'ab'
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 89e98ae61..ddf5724ae 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
-        continue_dl = info_dict.get('continuedl', False)
+        continue_dl = info_dict.get('continuedl', True)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

From 4747e2183acbddea61a3f10fb4ba1ed5b70c8bd1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 08:12:05 +0100
Subject: [PATCH 0098/1515] release 2015.03.28

---
 docs/supportedsites.md | 7 ++++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index baf7b3880..fd59cc2be 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -2,6 +2,8 @@
  - **1tv**: Первый канал
  - **1up.com**
  - **220.ro**
+ - **22tracks:genre**
+ - **22tracks:track**
  - **24video**
  - **3sat**
  - **4tube**
@@ -380,6 +382,8 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **safari**: safaribooksonline.com online video
+ - **safari:course**: safaribooksonline.com online courses
  - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
@@ -500,6 +504,7 @@
  - **Vbox7**
  - **VeeHD**
  - **Veoh**
+ - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
  - **VGTV**
@@ -588,7 +593,7 @@
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
- - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
  - **ZDF**
  - **ZDFChannel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 039ceadf2..dd93e295a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.24'
+__version__ = '2015.03.28'

From 643fe72717e6b9c45af4528e46dfc181cad7aebb Mon Sep 17 00:00:00 2001
From: Oskar Jauch <oskar.jauch@gmail.com>
Date: Sat, 28 Mar 2015 10:38:52 +0100
Subject: [PATCH 0099/1515] [DHM] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dhm.py      | 52 ++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/dhm.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d56eb6448..a65c0c25b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dbtv import DBTVIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
+from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
 from .dreisat import DreiSatIE
diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
new file mode 100644
index 000000000..d379c9d53
--- /dev/null
+++ b/youtube_dl/extractor/dhm.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import urllib2
+import xml.etree.ElementTree as ET
+import re
+
+
+class DHMIE(InfoExtractor):
+    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P<id>.*?)'
+
+    _TEST = {
+        'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
+        'md5': '11c475f670209bf6acca0b2b7ef51827',
+        'info_dict': {
+            'id': 'marshallwg',
+            'ext': 'flv',
+            'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE',
+            'thumbnail': 'http://www.dhm.de/filmarchiv/video/mpworkwg.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = ''
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'dc:title=\"(.*?)\"', webpage, 'title')
+
+        playlist_url = self._html_search_regex(
+            r'file: \'(.*?)\'', webpage, 'playlist URL')
+
+        xml_file = urllib2.urlopen(playlist_url)
+        data = xml_file.read()
+        xml_file.close()
+
+        root = ET.fromstring(data)
+        video_url = root[0][0][0].text
+        thumbnail = root[0][0][2].text
+
+        m = re.search('video/(.+?).flv', video_url)
+        if m:
+            video_id = m.group(1)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'thumbnail': thumbnail,
+        }

From ff79552f13efbfa87a814dd0a950aa797b08a5cd Mon Sep 17 00:00:00 2001
From: Oskar Jauch <oskar.jauch@gmail.com>
Date: Sat, 28 Mar 2015 10:42:35 +0100
Subject: [PATCH 0100/1515] [DHM] Add extractor description

---
 youtube_dl/extractor/dhm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index d379c9d53..a0a584f6a 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -9,6 +9,7 @@ import re
 
 
 class DHMIE(InfoExtractor):
+    IE_DESC = 'Deutsches Historisches Museum'
     _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P<id>.*?)'
 
     _TEST = {

From cb88671e37a9fbb964ee0b2ab46031195f2aab1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Mar 2015 14:18:11 +0100
Subject: [PATCH 0101/1515] [nbc] Recognize https urls (fixes #5300)

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3645d3033..80a01c778 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NBCIE(InfoExtractor):
-    _VALID_URL = r'http://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
+    _VALID_URL = r'https?://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
 
     _TESTS = [
         {

From af8c93086c45cf61cadc8571644713927659a65e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Mar 2015 22:30:13 +0600
Subject: [PATCH 0102/1515] [dhm] Simplify

---
 youtube_dl/extractor/dhm.py | 59 ++++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index a0a584f6a..80ee40018 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -1,53 +1,64 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-import urllib2
-import xml.etree.ElementTree as ET
-import re
+from ..utils import (
+    xpath_text,
+    parse_duration,
+)
 
 
 class DHMIE(InfoExtractor):
-    IE_DESC = 'Deutsches Historisches Museum'
-    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P<id>.*?)'
+    IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
+    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/die-filme/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
         'md5': '11c475f670209bf6acca0b2b7ef51827',
         'info_dict': {
-            'id': 'marshallwg',
+            'id': 'the-marshallplan-at-work-in-west-germany',
             'ext': 'flv',
             'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE',
-            'thumbnail': 'http://www.dhm.de/filmarchiv/video/mpworkwg.jpg',
+            'description': 'md5:1fabd480c153f97b07add61c44407c82',
+            'duration': 660,
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
-        video_id = ''
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'dc:title=\"(.*?)\"', webpage, 'title')
+        playlist_url = self._search_regex(
+            r"file\s*:\s*'([^']+)'", webpage, 'playlist url')
 
-        playlist_url = self._html_search_regex(
-            r'file: \'(.*?)\'', webpage, 'playlist URL')
+        playlist = self._download_xml(playlist_url, video_id)
 
-        xml_file = urllib2.urlopen(playlist_url)
-        data = xml_file.read()
-        xml_file.close()
+        track = playlist.find(
+            './{http://xspf.org/ns/0/}trackList/{http://xspf.org/ns/0/}track')
 
-        root = ET.fromstring(data)
-        video_url = root[0][0][0].text
-        thumbnail = root[0][0][2].text
+        video_url = xpath_text(
+            track, './{http://xspf.org/ns/0/}location',
+            'video url', fatal=True)
+        thumbnail = xpath_text(
+            track, './{http://xspf.org/ns/0/}image',
+            'thumbnail')
 
-        m = re.search('video/(.+?).flv', video_url)
-        if m:
-            video_id = m.group(1)
+        title = self._search_regex(
+            [r'dc:title="([^"]+)"', r'<title> &raquo;([^<]+)</title>'],
+            webpage, 'title').strip()
+        description = self._html_search_regex(
+            r'<p><strong>Description:</strong>(.+?)</p>',
+            webpage, 'description', fatal=False)
+        duration = parse_duration(self._search_regex(
+            r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
-            'title': title,
             'url': video_url,
+            'title': title,
+            'description': description,
+            'duration': duration,
             'thumbnail': thumbnail,
         }

From b7a2268e7b52fbedd1630ad101460d76cca9dcdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Mar 2015 23:43:15 +0600
Subject: [PATCH 0103/1515] Credit @ossi96 for dhm (#5305)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 59f1b5f21..b36086448 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -118,3 +118,4 @@ Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
+Oskar Jauch

From 5a3b315b5fcc1b1e153baeaed669e64d137d4047 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Mar 2015 23:55:15 +0600
Subject: [PATCH 0104/1515] [dhm] Improve _VALID_URL and add test

---
 youtube_dl/extractor/dhm.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 80ee40018..3ed1f1663 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 class DHMIE(InfoExtractor):
     IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
-    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/die-filme/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dhm\.de/filmarchiv/(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
         'md5': '11c475f670209bf6acca0b2b7ef51827',
         'info_dict': {
@@ -21,8 +21,17 @@ class DHMIE(InfoExtractor):
             'description': 'md5:1fabd480c153f97b07add61c44407c82',
             'duration': 660,
             'thumbnail': 're:^https?://.*\.jpg$',
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.dhm.de/filmarchiv/02-mapping-the-wall/peter-g/rolle-1/',
+        'md5': '09890226332476a3e3f6f2cb74734aa5',
+        'info_dict': {
+            'id': 'rolle-1',
+            'ext': 'flv',
+            'title': 'ROLLE 1',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -49,10 +58,10 @@ class DHMIE(InfoExtractor):
             webpage, 'title').strip()
         description = self._html_search_regex(
             r'<p><strong>Description:</strong>(.+?)</p>',
-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None)
         duration = parse_duration(self._search_regex(
             r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
-            webpage, 'duration', fatal=False))
+            webpage, 'duration', default=None))
 
         return {
             'id': video_id,

From 616af2f4b9458dd15152ddf6a7905f7818ae583a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Mar 2015 00:03:59 +0600
Subject: [PATCH 0105/1515] Unduplicate @ossi96

---
 AUTHORS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index b36086448..59f1b5f21 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -118,4 +118,3 @@ Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
-Oskar Jauch

From 5f88e028183896f67e35d2d82d9596213cd2c73e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 23:35:55 +0100
Subject: [PATCH 0106/1515] [ultimedia] PEP8

---
 youtube_dl/extractor/ultimedia.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 06554a1be..96c809eaf 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -42,7 +42,6 @@ class UltimediaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
 
         deliver_url = self._search_regex(
@@ -81,8 +80,8 @@ class UltimediaIE(InfoExtractor):
         title = clean_html((
             self._html_search_regex(
                 r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
-                webpage, 'title', default=None)
-            or self._search_regex(
+                webpage, 'title', default=None) or
+            self._search_regex(
                 r"var\s+nameVideo\s*=\s*'([^']+)'",
                 deliver_page, 'title')))
 

From ecb750a446dd4805467904864b99eab705e866a8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 23:39:41 +0100
Subject: [PATCH 0107/1515] [cnn] Match more URLs

---
 youtube_dl/extractor/cnn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 90ea07438..0a77e951c 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln)(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln|ktvk)(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
@@ -45,6 +45,9 @@ class CNNIE(InfoExtractor):
             'description': 'md5:e7223a503315c9f150acac52e76de086',
             'upload_date': '20141222',
         }
+    }, {
+        'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ad320e9b8376221a3eda935a358886b6e7ab7bf6 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 29 Mar 2015 04:57:37 +0300
Subject: [PATCH 0108/1515] [generic] Add support for 5min embeds (#5310)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8a49b0b54..042d23a13 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -620,6 +620,16 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
+        # 5min embed
+        {
+            'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
+            'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
+            'info_dict': {
+                'id': '518726732',
+                'ext': 'mp4',
+                'title': 'Facebook Creates "On This Day" | Crunch Report',
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1236,6 +1246,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Pladform')
 
+        # Look for 5min embeds
+        mobj = re.search(
+            r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
+        if mobj is not None:
+            return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 8343a03357ec4edb49b29bc841f2410ee4b610d1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Mar 2015 14:26:28 +0800
Subject: [PATCH 0109/1515] [douyutv] Fix extractor and improve error handling

---
 youtube_dl/extractor/douyutv.py | 57 ++++++++++++++++++++++++++-------
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index d7956e6e4..3a1665ab4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -1,19 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import hashlib
+import time
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (ExtractorError, unescapeHTML)
+from ..compat import (compat_str, compat_basestring)
 
 
 class DouyuTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
-            'id': 'iseven',
+            'id': '17732',
+            'display_id': 'iseven',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:9e525642c25a0a24302869937cf69d17',
+            'description': 'md5:c93d6692dde6fe33809a46edcbecca44',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'uploader_id': '431925',
@@ -22,22 +26,52 @@ class DouyuTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.douyutv.com/85982',
+        'info_dict': {
+            'id': '85982',
+            'display_id': '85982',
+            'ext': 'flv',
+            'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'douyu小漠',
+            'uploader_id': '3769985',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        if video_id.isdigit():
+            room_id = video_id
+        else:
+            page = self._download_webpage(url, video_id)
+            room_id = self._html_search_regex(
+                r'"room_id"\s*:\s*(\d+),', page, 'room id')
+
+        prefix = 'room/%s?aid=android&client_sys=android&time=%d' % (
+            room_id, int(time.time()))
+
+        auth = hashlib.md5((prefix + '1231').encode('ascii')).hexdigest()
         config = self._download_json(
-            'http://www.douyutv.com/api/client/room/%s' % video_id, video_id)
+            'http://www.douyutv.com/api/v1/%s&auth=%s' % (prefix, auth),
+            video_id)
 
         data = config['data']
 
         error_code = config.get('error', 0)
-        show_status = data.get('show_status')
         if error_code is not 0:
-            raise ExtractorError(
-                'Server reported error %i' % error_code, expected=True)
+            error_desc = 'Server reported error %i' % error_code
+            if isinstance(data, compat_str) or isinstance(data, compat_basestring):
+                error_desc += ': ' + data
+            raise ExtractorError(error_desc, expected=True)
 
+        show_status = data.get('show_status')
         # 1 = live, 2 = offline
         if show_status == '2':
             raise ExtractorError(
@@ -46,7 +80,7 @@ class DouyuTVIE(InfoExtractor):
         base_url = data['rtmp_url']
         live_path = data['rtmp_live']
 
-        title = self._live_title(data['room_name'])
+        title = self._live_title(unescapeHTML(data['room_name']))
         description = data.get('show_details')
         thumbnail = data.get('room_src')
 
@@ -66,7 +100,8 @@ class DouyuTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': room_id,
+            'display_id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 2ddf08358861a3c0f8724020c4eaf119e6be21a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Mar 2015 18:17:48 +0800
Subject: [PATCH 0110/1515] [douyutv] Simplify usage of isinstance

---
 youtube_dl/extractor/douyutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 3a1665ab4..479430c51 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -67,7 +67,7 @@ class DouyuTVIE(InfoExtractor):
         error_code = config.get('error', 0)
         if error_code is not 0:
             error_desc = 'Server reported error %i' % error_code
-            if isinstance(data, compat_str) or isinstance(data, compat_basestring):
+            if isinstance(data, (compat_str, compat_basestring)):
                 error_desc += ': ' + data
             raise ExtractorError(error_desc, expected=True)
 

From 9b4774b21bb26eea9eb8042b2ff248a176595c5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Mar 2015 20:40:05 +0800
Subject: [PATCH 0111/1515] [Xuite] Fix extraction on python 3.2

base64.b64decode() accept only binary types in Python 3.2
---
 youtube_dl/extractor/xuite.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 4971965f9..81d885fdc 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -69,18 +69,26 @@ class XuiteIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def base64_decode_utf8(data):
+        return base64.b64decode(data.encode('utf-8')).decode('utf-8')
+
+    @staticmethod
+    def base64_encode_utf8(data):
+        return base64.b64encode(data.encode('utf-8')).decode('utf-8')
+
     def _extract_flv_config(self, media_id):
-        base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
+        base64_media_id = self.base64_encode_utf8(media_id)
         flv_config = self._download_xml(
             'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id,
             'flv config')
         prop_dict = {}
         for prop in flv_config.findall('./property'):
-            prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
+            prop_id = self.base64_decode_utf8(prop.attrib['id'])
             # CDATA may be empty in flv config
             if not prop.text:
                 continue
-            encoded_content = base64.b64decode(prop.text).decode('utf-8')
+            encoded_content = self.base64_decode_utf8(prop.text)
             prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
         return prop_dict
 

From 4d5d14f5cf096f4bd90ad373cad687dc82bf8a8f Mon Sep 17 00:00:00 2001
From: Joram Schrijver <i@joram.io>
Date: Sun, 29 Mar 2015 23:41:06 +0200
Subject: [PATCH 0112/1515] [Dumpert] Add new extractor

Add support for the Dutch video site Dumpert. http://www.dumpert.nl/
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dumpert.py  | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/dumpert.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a65c0c25b..43bac0252 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -115,6 +115,7 @@ from .drtuber import DrTuberIE
 from .drtv import DRTVIE
 from .dvtv import DVTVIE
 from .dump import DumpIE
+from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .divxstage import DivxStageIE
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
new file mode 100644
index 000000000..52d07deac
--- /dev/null
+++ b/youtube_dl/extractor/dumpert.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+
+from .common import InfoExtractor
+
+
+class DumpertIE(InfoExtractor):
+    _VALID_URL = (r'https?://(?:www\.)?dumpert\.nl/mediabase/'
+                  r'(?P<id>[0-9]+/[0-9a-zA-Z]+)/?.*')
+    _TEST = {
+        'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
+        'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
+        'info_dict': {
+            'id': '6646981/951bc60f',
+            'ext': 'mp4',
+            'title': 'Ik heb nieuws voor je',
+            'description': 'Niet schrikken hoor'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta('title', webpage)
+        description = self._html_search_meta('description', webpage)
+
+        files_base64 = self._html_search_regex(r'data-files="(.*?)"',
+                                               webpage,
+                                               'files')
+        files_json = base64.b64decode(files_base64).decode('iso-8859-1')
+        files = self._parse_json(files_json, video_id)
+
+        format_names = ['flv', 'mobile', 'tablet', '720p']
+        formats = [{'format_id': name,
+                    'url': files[name].replace(r'\/', '/')}
+                   for name in format_names
+                   if name in files]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats
+        }

From 7700207ec7d39e1594d9963a5014ddcb30c7301a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 19:41:04 +0600
Subject: [PATCH 0113/1515] [pornhub] Fix comment count extraction (Closes
 #5320)

---
 youtube_dl/extractor/pornhub.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3a27e3789..0c8b731cf 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -33,10 +33,8 @@ class PornHubIE(InfoExtractor):
     }
 
     def _extract_count(self, pattern, webpage, name):
-        count = self._html_search_regex(pattern, webpage, '%s count' % name, fatal=False)
-        if count:
-            count = str_to_int(count)
-        return count
+        return str_to_int(self._search_regex(
+            pattern, webpage, '%s count' % name, fatal=False))
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -62,11 +60,14 @@ class PornHubIE(InfoExtractor):
         if thumbnail:
             thumbnail = compat_urllib_parse.unquote(thumbnail)
 
-        view_count = self._extract_count(r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
-        like_count = self._extract_count(r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
-        dislike_count = self._extract_count(r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
+        view_count = self._extract_count(
+            r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
+        like_count = self._extract_count(
+            r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
+        dislike_count = self._extract_count(
+            r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
         comment_count = self._extract_count(
-            r'All comments \(<var class="videoCommentCount">([\d,\.]+)</var>', webpage, 'comment')
+            r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
         video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:

From 87270c84166bb014ba1043a6eeee7330694b0649 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 20:11:51 +0600
Subject: [PATCH 0114/1515] [dumpert] Simplify and fix python 3.2

---
 youtube_dl/extractor/dumpert.py | 39 ++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 52d07deac..e43bc81b2 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
+from ..utils import qualities
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = (r'https?://(?:www\.)?dumpert\.nl/mediabase/'
-                  r'(?P<id>[0-9]+/[0-9a-zA-Z]+)/?.*')
+    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/mediabase/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
     _TEST = {
         'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -16,7 +16,8 @@ class DumpertIE(InfoExtractor):
             'id': '6646981/951bc60f',
             'ext': 'mp4',
             'title': 'Ik heb nieuws voor je',
-            'description': 'Niet schrikken hoor'
+            'description': 'Niet schrikken hoor',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -24,24 +25,32 @@ class DumpertIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_meta('title', webpage)
-        description = self._html_search_meta('description', webpage)
+        files_base64 = self._search_regex(
+            r'data-files="([^"]+)"', webpage, 'data files')
 
-        files_base64 = self._html_search_regex(r'data-files="(.*?)"',
-                                               webpage,
-                                               'files')
-        files_json = base64.b64decode(files_base64).decode('iso-8859-1')
-        files = self._parse_json(files_json, video_id)
+        files = self._parse_json(
+            base64.b64decode(files_base64.encode('utf-8')).decode('utf-8'),
+            video_id)
 
-        format_names = ['flv', 'mobile', 'tablet', '720p']
-        formats = [{'format_id': name,
-                    'url': files[name].replace(r'\/', '/')}
-                   for name in format_names
-                   if name in files]
+        quality = qualities(['flv', 'mobile', 'tablet', '720p'])
+
+        formats = [{
+            'url': video_url,
+            'format_id': format_id,
+            'quality': quality(format_id),
+        } for format_id, video_url in files.items() if format_id != 'still']
+        self._sort_formats(formats)
+
+        title = self._html_search_meta(
+            'title', webpage) or self._og_search_title(webpage)
+        description = self._html_search_meta(
+            'description', webpage) or self._og_search_description(webpage)
+        thumbnail = files.get('still') or self._og_search_thumbnail(webpage)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'formats': formats
         }

From fd203fe35774f6e8a5ce17ad78650089d2536c97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 20:12:55 +0600
Subject: [PATCH 0115/1515] Credit @jorams or dumpert.nl (#5319)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 59f1b5f21..48be31e29 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -118,3 +118,4 @@ Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
+Joram Schrijver

From c808ef81bb67f737b89671ce882abfca666e0139 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 21:03:38 +0600
Subject: [PATCH 0116/1515] [soundcloud:set:user] Support mobile URLs (Closes
 #5323)

---
 youtube_dl/extractor/soundcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 9d4505972..316b2c90f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -242,7 +242,7 @@ class SoundcloudIE(InfoExtractor):
 
 
 class SoundcloudSetIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
     IE_NAME = 'soundcloud:set'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
@@ -287,7 +287,7 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band',

From edd7344820303e550c9daae1ef591be8068ac47e Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 30 Mar 2015 18:15:08 +0300
Subject: [PATCH 0117/1515] [phoenix] Extend _VALID_URL (#5322)

---
 youtube_dl/extractor/phoenix.py | 40 ++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index a20672c0c..46cebc0d7 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -5,19 +5,33 @@ from .zdf import extract_from_xml_url
 
 
 class PhoenixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?phoenix\.de/content/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.phoenix.de/content/884301',
-        'md5': 'ed249f045256150c92e72dbb70eadec6',
-        'info_dict': {
-            'id': '884301',
-            'ext': 'mp4',
-            'title': 'Michael Krons mit Hans-Werner Sinn',
-            'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
-            'upload_date': '20141025',
-            'uploader': 'Im Dialog',
-        }
-    }
+    _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
+        (?:
+            phoenix/die_sendungen/(?:[^/]+/)?
+        )?
+        (?P<id>[0-9]+)'''
+    _TESTS = [
+        {
+            'url': 'http://www.phoenix.de/content/884301',
+            'md5': 'ed249f045256150c92e72dbb70eadec6',
+            'info_dict': {
+                'id': '884301',
+                'ext': 'mp4',
+                'title': 'Michael Krons mit Hans-Werner Sinn',
+                'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
+                'upload_date': '20141025',
+                'uploader': 'Im Dialog',
+            }
+        },
+        {
+            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/869815',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/diskussionen/928234',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From a28ccbabc60c81016c851ae46365be377ea83795 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 02:21:27 +0800
Subject: [PATCH 0118/1515] [Yahoo/NBCSports] Fix #5226

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 21 +++++++++++++++++++++
 youtube_dl/extractor/yahoo.py    | 14 ++++++++++++++
 3 files changed, 36 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 43bac0252..5d0d2a9bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -310,6 +310,7 @@ from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
     NBCIE,
+    NBCSportsIE,
     NBCNewsIE,
 )
 from .ndr import NDRIE
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 80a01c778..033bf71f0 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -50,6 +50,27 @@ class NBCIE(InfoExtractor):
         return self.url_result(theplatform_url)
 
 
+class NBCSportsIE(InfoExtractor):
+    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
+        'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
+        'info_dict': {
+            'id': '9CsDKds0kvHI',
+            'ext': 'flv',
+            'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
+            'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        theplatform_url = self._og_search_video_url(webpage)
+        return self.url_result(theplatform_url, 'ThePlatform')
+
+
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 97dbac4cc..6e72f1e55 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -129,6 +129,15 @@ class YahooIE(InfoExtractor):
         }, {
             'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
             'only_matching': True,
+        }, {
+            'note': 'NBC Sports embeds',
+            'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
+            'info_dict': {
+                'id': '9CsDKds0kvHI',
+                'ext': 'flv',
+                'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
+                'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
+            }
         }
     ]
 
@@ -151,6 +160,11 @@ class YahooIE(InfoExtractor):
                 items = json.loads(items_json)
                 video_id = items[0]['id']
                 return self._get_info(video_id, display_id, webpage)
+        # Look for NBCSports iframes
+        iframe_m = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
+        if iframe_m:
+            return self.url_result(iframe_m.group('url'), 'NBCSports')
 
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,

From a2a4d5fa313d5244d24fa70d5db91971a7583d79 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 02:47:18 +0800
Subject: [PATCH 0119/1515] [Yahoo/NBCSports] Generalize NBC sports info
 extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 31 ++++++++++++++++++++++++++++++-
 youtube_dl/extractor/yahoo.py    | 10 ++++++----
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d0d2a9bc..b113aaec6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -310,6 +310,7 @@ from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
     NBCIE,
+    NBCSportsVPlayerIE,
     NBCSportsIE,
     NBCNewsIE,
 )
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 033bf71f0..c8dd72ab4 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -50,7 +50,7 @@ class NBCIE(InfoExtractor):
         return self.url_result(theplatform_url)
 
 
-class NBCSportsIE(InfoExtractor):
+class NBCSportsVPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
 
     _TEST = {
@@ -64,6 +64,13 @@ class NBCSportsIE(InfoExtractor):
         }
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        iframe_m = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
+        if iframe_m:
+            return iframe_m.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -71,6 +78,28 @@ class NBCSportsIE(InfoExtractor):
         return self.url_result(theplatform_url, 'ThePlatform')
 
 
+class NBCSportsIE(InfoExtractor):
+    # Does not include https becuase its certificate is invalid
+    _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
+
+    _TEST = {
+        'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
+        'md5': 'ba6c93f96b67bf05344f78bd523dac0f',
+        'info_dict': {
+            'id': 'PHJSaFWbrTY9',
+            'ext': 'flv',
+            'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
+            'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        return self.url_result(
+            NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')
+
+
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 6e72f1e55..43776d1e6 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -17,6 +17,8 @@ from ..utils import (
     int_or_none,
 )
 
+from .nbc import NBCSportsVPlayerIE
+
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
@@ -132,6 +134,7 @@ class YahooIE(InfoExtractor):
         }, {
             'note': 'NBC Sports embeds',
             'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
+            'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
             'info_dict': {
                 'id': '9CsDKds0kvHI',
                 'ext': 'flv',
@@ -161,10 +164,9 @@ class YahooIE(InfoExtractor):
                 video_id = items[0]['id']
                 return self._get_info(video_id, display_id, webpage)
         # Look for NBCSports iframes
-        iframe_m = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
-        if iframe_m:
-            return self.url_result(iframe_m.group('url'), 'NBCSports')
+        nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
+        if nbc_sports_url:
+            return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,

From 1d31e7a2fc2fb78c792754578a8a58b056811b84 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 02:51:11 +0800
Subject: [PATCH 0120/1515] [NBCSports] Move imports alphabetically

---
 youtube_dl/extractor/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b113aaec6..9fddb8e32 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -310,9 +310,9 @@ from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
     NBCIE,
-    NBCSportsVPlayerIE,
-    NBCSportsIE,
     NBCNewsIE,
+    NBCSportsIE,
+    NBCSportsVPlayerIE,
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE

From a2edf2e7ff314eaa3124c1da1b962d054b6d9fff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 03:36:09 +0800
Subject: [PATCH 0121/1515] [NBC/ThePlatform/Generic] Add a generic detector
 for NBCSportsVPlayer and enhance error detection in ThePlatformIE

---
 youtube_dl/extractor/generic.py     | 15 +++++++++++++++
 youtube_dl/extractor/nbc.py         |  2 +-
 youtube_dl/extractor/theplatform.py |  2 +-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 042d23a13..9ddf36f6b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,6 +29,7 @@ from ..utils import (
     xpath_text,
 )
 from .brightcove import BrightcoveIE
+from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
 from .smotri import SmotriIE
@@ -639,6 +640,15 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150228',
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
+        },
+        # NBC Sports vplayer embeds
+        {
+            'url': 'http://bbs.clutchfans.net/showthread.php?t=244180',
+            'info_dict': {
+                'id': '_hqLjQ95yx8Z',
+                'ext': 'flv'
+            },
+            'skip': 'This content expired on 9/17/14 12:23 PM',
         }
     ]
 
@@ -1252,6 +1262,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
 
+        # Look for NBC Sports VPlayer embeds
+        nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
+        if nbc_sports_url:
+            return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index c8dd72ab4..be9969d12 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -51,7 +51,7 @@ class NBCIE(InfoExtractor):
 
 
 class NBCSportsVPlayerIE(InfoExtractor):
-    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
     _TEST = {
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index feac666f7..0e3e627f4 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -92,7 +92,7 @@ class ThePlatformIE(InfoExtractor):
             error_msg = next(
                 n.attrib['abstract']
                 for n in meta.findall(_x('.//smil:ref'))
-                if n.attrib.get('title') == 'Geographic Restriction')
+                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
         except StopIteration:
             pass
         else:

From 5cbb2699ee04535449e37a07dd9cac9bfd224fe3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 03:38:45 +0800
Subject: [PATCH 0122/1515] [NBCSports] Add a test case for extended _VALID_URL

---
 youtube_dl/extractor/nbc.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index be9969d12..395f53df3 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -53,7 +53,7 @@ class NBCIE(InfoExtractor):
 class NBCSportsVPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
         'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
         'info_dict': {
@@ -62,7 +62,11 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
         }
-    }
+    }, {
+        'note': 'This video is already expired. It\'s for testing _VALID_URL',
+        'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def _extract_url(webpage):

From e15307a612ea588b504f1f03ba0201612df66b35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 13:13:29 +0800
Subject: [PATCH 0123/1515] [NBCSports/Yahoo] Comment out some MD5 checksums

They seems to change constantly
---
 youtube_dl/extractor/nbc.py   | 2 --
 youtube_dl/extractor/yahoo.py | 1 -
 2 files changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 395f53df3..b7f6a5366 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -55,7 +55,6 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
-        'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
         'info_dict': {
             'id': '9CsDKds0kvHI',
             'ext': 'flv',
@@ -88,7 +87,6 @@ class NBCSportsIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
-        'md5': 'ba6c93f96b67bf05344f78bd523dac0f',
         'info_dict': {
             'id': 'PHJSaFWbrTY9',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 43776d1e6..b777159c5 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -134,7 +134,6 @@ class YahooIE(InfoExtractor):
         }, {
             'note': 'NBC Sports embeds',
             'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
-            'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
             'info_dict': {
                 'id': '9CsDKds0kvHI',
                 'ext': 'flv',

From facecb84a11ba59a67d01c659ec3944179774418 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Mar 2015 20:11:14 +0600
Subject: [PATCH 0124/1515] [generic] Add working NBC Sports vplayer test

---
 youtube_dl/extractor/generic.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9ddf36f6b..2ff002643 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -641,14 +641,15 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
-        # NBC Sports vplayer embeds
+        # NBC Sports vplayer embed
         {
-            'url': 'http://bbs.clutchfans.net/showthread.php?t=244180',
+            'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
             'info_dict': {
-                'id': '_hqLjQ95yx8Z',
-                'ext': 'flv'
+                'id': 'ln7x1qSThw4k',
+                'ext': 'flv',
+                'title': "PFT Live: New leader in the 'new-look' defense",
+                'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
             },
-            'skip': 'This content expired on 9/17/14 12:23 PM',
         }
     ]
 

From c89fbfb385c4989af3bf2eb45e300e01c385cfc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Mar 2015 20:14:37 +0600
Subject: [PATCH 0125/1515] [nbc] Remove redundant note

This is already supposed by `only_matching`
---
 youtube_dl/extractor/nbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index b7f6a5366..ecd0ac8b1 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -62,7 +62,6 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
         }
     }, {
-        'note': 'This video is already expired. It\'s for testing _VALID_URL',
         'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
         'only_matching': True,
     }]

From 2a0c2ca2b8be753f13f9b9dfdcce55560e4953e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Mar 2015 20:55:21 +0600
Subject: [PATCH 0126/1515] [dailymotion] Fix ff cookie and use it for embed
 page (Closes #5330)

---
 youtube_dl/extractor/dailymotion.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 4f67c3aac..47d58330b 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -25,8 +25,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
     def _build_request(url):
         """Build a request with the family filter disabled"""
         request = compat_urllib_request.Request(url)
-        request.add_header('Cookie', 'family_filter=off')
-        request.add_header('Cookie', 'ff=off')
+        request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
 
@@ -112,8 +111,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 
         embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
-        embed_page = self._download_webpage(embed_url, video_id,
-                                            'Downloading embed page')
+        embed_request = self._build_request(embed_url)
+        embed_page = self._download_webpage(
+            embed_request, video_id, 'Downloading embed page')
         info = self._search_regex(r'var info = ({.*?}),$', embed_page,
                                   'video info', flags=re.MULTILINE)
         info = json.loads(info)

From 55cde6ef3c046dfd5f6ef84b908d38ee4375cf08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Apr 2015 22:02:55 +0600
Subject: [PATCH 0127/1515] [varzesh3] Simplify

---
 youtube_dl/extractor/varzesh3.py | 43 +++++++++++++++-----------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index eb49586cc..9369abaf8 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -1,48 +1,45 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
-import re
 
 
 class Varzesh3IE(InfoExtractor):
-    _VALID_URL = r'(?P<url>(https?://(?:www\.)?video\.varzesh3\.com)/(?P<id>.+))'
-    _TEST ={
+    _VALID_URL = r'https?://(?:www\.)?video\.varzesh3\.com/(?:[^/]+/)+(?P<id>[^/]+)/?'
+    _TEST = {
         'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
         'md5': '2a933874cb7dce4366075281eb49e855',
         'info_dict': {
-            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/video/namayeshi/saves_week26.mp4',
             'id': '76337',
             'ext': 'mp4',
             'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
-            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
             'description': 'فصل ۲۰۱۵-۲۰۱۴',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        if not 'shortlink' in webpage:
-            raise ExtractorError('URL has no videos or there is a problem.')
+        webpage = self._download_webpage(url, display_id)
 
-        title = self._html_search_regex(r'meta[^>]+property="og:title"[^>]+content="([^"]+)"', webpage, 'title')
-        video_link = self._html_search_regex(r'source[^>]+src="([^"]+)"', webpage, 'video_link')
-        vid_id = self._html_search_regex(r"link[^>]+rel='canonical'[^>]+href='\/\?p=([^']+)'\/>", webpage, 'vid_id')
-        try:
-            description = self._html_search_regex(r'<div class="matn">(.*?)</div>', webpage, 'description', flags=re.DOTALL)
-        except:
-            description = title
-        thumbnail = self._html_search_regex(r'link[^>]+rel="image_src"[^>]+href="([^"]+)"', webpage, 'thumbnail')
+        video_url = self._search_regex(
+            r'<source[^>]+src="([^"]+)"', webpage, 'video url')
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(
+            r'(?s)<div class="matn">(.+?)</div>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        video_id = self._search_regex(
+            r"<link[^>]+rel='(?:canonical|shortlink)'[^>]+href='/\?p=([^']+)'",
+            webpage, display_id, default=display_id)
 
         return {
-            'url': video_link,
-            'id': vid_id,
+            'url': video_url,
+            'id': video_id,
             'title': title,
-            'ext': video_link.split(".")[-1],
             'description': description,
             'thumbnail': thumbnail,
         }

From 1a944d8a2ae8756b78a709862cccae58720445ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 2 Apr 2015 14:09:55 +0200
Subject: [PATCH 0128/1515] Print a warning if no ssl certificates are loaded

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 640b8c99d..4fa2223ad 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1768,6 +1768,12 @@ class YoutubeDL(object):
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
+        # The ssl context is only available in python 2.7.9 and 3.x
+        if hasattr(https_handler, '_context'):
+            if len(https_handler._context.get_ca_certs()) == 0:
+                self.report_warning(
+                    'No ssl certificates were loaded, urls that use https '
+                    'won\'t work')
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             proxy_handler, https_handler, cookie_processor, ydlh)

From 8075d4f99dbbf330c4a44d58b535055853b99aab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 20:26:05 +0600
Subject: [PATCH 0129/1515] [playfm] Adapt to v2api (Closes #5344)

---
 youtube_dl/extractor/playfm.py | 87 +++++++++++++++-------------------
 1 file changed, 37 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/playfm.py b/youtube_dl/extractor/playfm.py
index 9576aed0e..e766ccca3 100644
--- a/youtube_dl/extractor/playfm.py
+++ b/youtube_dl/extractor/playfm.py
@@ -4,85 +4,72 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    float_or_none,
     int_or_none,
-    str_to_int,
+    parse_iso8601,
 )
 
 
 class PlayFMIE(InfoExtractor):
     IE_NAME = 'play.fm'
-    _VALID_URL = r'https?://(?:www\.)?play\.fm/[^?#]*(?P<upload_date>[0-9]{8})(?P<id>[0-9]{6})(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?play\.fm/(?P<slug>(?:[^/]+/)+(?P<id>[^/]+))/?(?:$|[?#])'
 
     _TEST = {
-        'url': 'http://www.play.fm/recording/leipzigelectronicmusicbatofarparis_fr20140712137220',
+        'url': 'https://www.play.fm/dan-drastic/sven-tasnadi-leipzig-electronic-music-batofar-paris-fr-2014-07-12',
         'md5': 'c505f8307825a245d0c7ad1850001f22',
         'info_dict': {
-            'id': '137220',
+            'id': '71276',
             'ext': 'mp3',
-            'title': 'LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
-            'uploader': 'Sven Tasnadi',
-            'uploader_id': 'sventasnadi',
-            'duration': 5627.428,
-            'upload_date': '20140712',
+            'title': 'Sven Tasnadi - LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
+            'description': '',
+            'duration': 5627,
+            'timestamp': 1406033781,
+            'upload_date': '20140722',
+            'uploader': 'Dan Drastic',
+            'uploader_id': '71170',
             'view_count': int,
             'comment_count': int,
-            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        upload_date = mobj.group('upload_date')
+        slug = mobj.group('slug')
 
-        rec_data = compat_urllib_parse.urlencode({'rec_id': video_id})
-        req = compat_urllib_request.Request(
-            'http://www.play.fm/flexRead/recording', data=rec_data)
-        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        rec_doc = self._download_xml(req, video_id)
+        recordings = self._download_json(
+            'http://v2api.play.fm/recordings/slug/%s' % slug, video_id)
 
-        error_node = rec_doc.find('./error')
-        if error_node is not None:
-            raise ExtractorError('An error occured: %s (code %s)' % (
-                error_node.text, rec_doc.find('./status').text))
+        error = recordings.get('error')
+        if isinstance(error, dict):
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error.get('message')),
+                expected=True)
 
-        recording = rec_doc.find('./recording')
-        title = recording.find('./title').text
-        view_count = str_to_int(recording.find('./stats/playcount').text)
-        comment_count = str_to_int(recording.find('./stats/comments').text)
-        duration = float_or_none(recording.find('./duration').text, scale=1000)
-        thumbnail = recording.find('./image').text
-
-        artist = recording.find('./artists/artist')
-        uploader = artist.find('./name').text
-        uploader_id = artist.find('./slug').text
-
-        video_url = '%s//%s/%s/%s/offset/0/sh/%s/rec/%s/jingle/%s/loc/%s' % (
-            'http:', recording.find('./url').text,
-            recording.find('./_class').text, recording.find('./file_id').text,
-            rec_doc.find('./uuid').text, video_id,
-            rec_doc.find('./jingle/file_id').text,
-            'http%3A%2F%2Fwww.play.fm%2Fplayer',
-        )
+        audio_url = recordings['audio']
+        video_id = compat_str(recordings.get('id') or video_id)
+        title = recordings['title']
+        description = recordings.get('description')
+        duration = int_or_none(recordings.get('recordingDuration'))
+        timestamp = parse_iso8601(recordings.get('created_at'))
+        uploader = recordings.get('page', {}).get('title')
+        uploader_id = compat_str(recordings.get('page', {}).get('id'))
+        view_count = int_or_none(recordings.get('playCount'))
+        comment_count = int_or_none(recordings.get('commentCount'))
+        categories = [tag['name'] for tag in recordings.get('tags', []) if tag.get('name')]
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'mp3',
-            'filesize': int_or_none(recording.find('./size').text),
+            'url': audio_url,
             'title': title,
-            'upload_date': upload_date,
-            'view_count': view_count,
-            'comment_count': comment_count,
+            'description': description,
             'duration': duration,
-            'thumbnail': thumbnail,
+            'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'categories': categories,
         }

From f5b669113fc530922280987ff9e19afa780b8844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:32:16 +0600
Subject: [PATCH 0130/1515] [miomio] Simplify and fix python 2.6 issue

---
 youtube_dl/extractor/miomio_tv.py | 101 ++++++++++++++++++------------
 1 file changed, 60 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio_tv.py
index ae20a32fa..dc2ba7cb4 100644
--- a/youtube_dl/extractor/miomio_tv.py
+++ b/youtube_dl/extractor/miomio_tv.py
@@ -1,74 +1,93 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
+
 from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    int_or_none,
+)
 
 
 class MiomioTvIE(InfoExtractor):
     IE_NAME = 'miomio.tv'
     _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.miomio.tv/watch/cc179734/',
         'md5': '48de02137d0739c15b440a224ad364b9',
         'info_dict': {
             'id': '179734',
-            'title': u'\u624b\u7ed8\u52a8\u6f2b\u9b3c\u6ce3\u4f46\u4e01\u5168\u7a0b\u753b\u6cd5',
-            'ext': 'flv'
-        }
-    }
+            'ext': 'flv',
+            'title': '手绘动漫鬼泣但丁全程画法',
+            'duration': 354,
+        },
+    }, {
+        'url': 'http://www.miomio.tv/watch/cc184024/',
+        'info_dict': {
+            'id': '43729',
+            'title': '《动漫同人插画绘制》',
+        },
+        'playlist_mincount': 86,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<meta\s+name="description"\s+content="\s*([^"]*)\s*"', webpage, 'title')
-        ref_path = self._search_regex(r'src="(/mioplayer/.*?)"', webpage, 'ref_path')
-        referer = 'http://www.miomio.tv{0}'.format(ref_path)
-        xml_config = self._search_regex(r'flashvars="type=sina&amp;(.*?)&amp;cid=', webpage, 'xml config')
+        title = self._html_search_meta(
+            'description', webpage, 'title', fatal=True)
+
+        mioplayer_path = self._search_regex(
+            r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
+
+        xml_config = self._search_regex(
+            r'flashvars="type=sina&amp;(.+?)&amp;',
+            webpage, 'xml config')
         
         # skipping the following page causes lags and eventually connection drop-outs
-        # id is normally a rotating three digit value but a fixed value always appears to work
-        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={0}&r=cc{1}".format(id, 945), video_id)
+        self._request_webpage(
+            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
+            video_id)
 
         # the following xml contains the actual configuration information on the video file(s)
-        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config)
-        vidconfig = self._download_xml(xml_url, video_id)
-
-        file_els = vidconfig.findall('.//durl')
-
-        entries = []
-
-        for file_el in file_els:
-            segment_id = file_el.find('order').text.strip()
-            segment_title = '_'.join([title, segment_id])
-            segment_duration = file_el.find('length').text.strip()
-            segment_url = file_el.find('url').text.strip()
-
-            entries.append({
-                'id': segment_id,
-                'title': segment_title,
-                'duration': segment_duration,
-                'url': segment_url
-            })
+        vid_config = self._download_xml(
+            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
+            video_id)
 
         http_headers = {
-            'Referer': referer,
-            'Accept-Encoding': 'gzip, deflate',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+            'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
         }
 
+        entries = []
+        for f in vid_config.findall('./durl'):
+            segment_url = xpath_text(f, 'url', 'video url')
+            if not segment_url:
+                continue
+            order = xpath_text(f, 'order', 'order')
+            segment_id = video_id
+            segment_title = title
+            if order:
+                segment_id += '-%s' % order
+                segment_title += ' part %s' % order
+            entries.append({
+                'id': segment_id,
+                'url': segment_url,
+                'title': segment_title,
+                'duration': int_or_none(xpath_text(f, 'length', 'duration'), 1000),
+                'http_headers': http_headers,
+            })
+
         if len(entries) == 1:
-            return {
-                'id': video_id,
-                'title': title,
-                'url': entries[0]['url'],
-                'http_headers': http_headers
-            }
+            segment = entries[0]
+            segment['id'] = video_id
+            segment['title'] = title
+            return segment
 
         return {
             '_type': 'multi_video',
             'id': video_id,
-            'title': title,
             'entries': entries,
-            'http_headers': http_headers
+            'title': title,
+            'http_headers': http_headers,
         }

From e03bfb30ce0062c048759ed01f5a2fe4190d66fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:33:30 +0600
Subject: [PATCH 0131/1515] [miomio] Rename extractor

---
 youtube_dl/extractor/__init__.py                 | 2 +-
 youtube_dl/extractor/{miomio_tv.py => miomio.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{miomio_tv.py => miomio.py} (98%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0b9736f2d..9700d81f5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
-from .miomio_tv import MiomioTvIE
+from .miomio import MioMioIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio.py
similarity index 98%
rename from youtube_dl/extractor/miomio_tv.py
rename to youtube_dl/extractor/miomio.py
index dc2ba7cb4..11608f730 100644
--- a/youtube_dl/extractor/miomio_tv.py
+++ b/youtube_dl/extractor/miomio.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class MiomioTvIE(InfoExtractor):
+class MioMioIE(InfoExtractor):
     IE_NAME = 'miomio.tv'
     _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
     _TESTS = [{

From 2ec8e04cac895121a71f11a44b855b1bf8a0195e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:34:08 +0600
Subject: [PATCH 0132/1515] [miomio] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9700d81f5..aae4aae4c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,9 +274,9 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
-from .miomio import MioMioIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
+from .miomio import MioMioIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
 from .mixcloud import MixcloudIE

From a3c7019e065298103026960e00fff6b80b6bcf62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:50:10 +0600
Subject: [PATCH 0133/1515] [YoutubeDL] Check for `get_ca_certs` availability

`get_ca_certs` is not available in python <3.4
---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4fa2223ad..ce4b72fd3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1770,7 +1770,9 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         # The ssl context is only available in python 2.7.9 and 3.x
         if hasattr(https_handler, '_context'):
-            if len(https_handler._context.get_ca_certs()) == 0:
+            ctx = https_handler._context
+            # get_ca_certs is unavailable prior to python 3.4
+            if hasattr(ctx, 'get_ca_certs') and len(ctx.get_ca_certs()) == 0:
                 self.report_warning(
                     'No ssl certificates were loaded, urls that use https '
                     'won\'t work')

From 6b7556a554a490a21307fec65cdaefc3331027c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Apr 2015 01:47:18 +0600
Subject: [PATCH 0134/1515] Credit @tiktok7 for miomio.tv (#5265)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 48be31e29..48769320a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -119,3 +119,4 @@ Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
 Joram Schrijver
+Will W.

From a9cbab173584c716219b348ff36ccd5274f75249 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 3 Apr 2015 10:22:25 +0200
Subject: [PATCH 0135/1515] release 2015.04.03

---
 docs/supportedsites.md | 6 ++++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fd59cc2be..2785b9587 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -111,6 +111,7 @@
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
+ - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
@@ -120,6 +121,7 @@
  - **DrTuber**
  - **DRTV**
  - **Dump**
+ - **Dumpert**
  - **dvtv**: http://video.aktualne.cz/
  - **EaglePlatform**
  - **EbaumsWorld**
@@ -253,6 +255,7 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
+ - **miomio.tv**
  - **mitele.es**
  - **mixcloud**
  - **MLB**
@@ -286,6 +289,8 @@
  - **NBA**
  - **NBC**
  - **NBCNews**
+ - **NBCSports**
+ - **NBCSportsVPlayer**
  - **ndr**: NDR.de - Mediathek
  - **NDTV**
  - **NerdCubedFeed**
@@ -501,6 +506,7 @@
  - **Urort**: NRK P3 Urørt
  - **ustream**
  - **ustream:channel**
+ - **Varzesh3**
  - **Vbox7**
  - **VeeHD**
  - **Veoh**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dd93e295a..e1c385bec 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.28'
+__version__ = '2015.04.03'

From 4bbeb19fc77a49af763ce3443293b29b8450d686 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 14:09:07 +0200
Subject: [PATCH 0136/1515] [miomio] pep8: remove whitespaces in empty line

---
 youtube_dl/extractor/miomio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 11608f730..cc3f27194 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -44,7 +44,7 @@ class MioMioIE(InfoExtractor):
         xml_config = self._search_regex(
             r'flashvars="type=sina&amp;(.+?)&amp;',
             webpage, 'xml config')
-        
+
         # skipping the following page causes lags and eventually connection drop-outs
         self._request_webpage(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),

From 3da4b31359c8dbbad3477fbe8341e6fc293cda82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 14:09:50 +0200
Subject: [PATCH 0137/1515] [postprocessor/ffmpeg] Fix crash when
 ffprobe/avprobe are not installed (closes #5349)

'self.probe_basename' was None, so 'probe_executable' raised a KeyError exception
---
 youtube_dl/postprocessor/ffmpeg.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 55adf9685..0b60ac7e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -116,6 +116,10 @@ class FFmpegPostProcessor(PostProcessor):
     def executable(self):
         return self._paths[self.basename]
 
+    @property
+    def probe_available(self):
+        return self.probe_basename is not None
+
     @property
     def probe_executable(self):
         return self._paths[self.probe_basename]
@@ -168,7 +172,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
     def get_audio_codec(self, path):
 
-        if not self.probe_executable:
+        if not self.probe_available:
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [

From ff2be6e180f1af471dd6d533719d9c595c756557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 15:01:17 +0200
Subject: [PATCH 0138/1515] [bloomberg] Adapt to website changes (fixes #5347)

---
 youtube_dl/extractor/bloomberg.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 4a88ccd13..0dca29b71 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,32 +6,39 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://www\.bloomberg\.com/news/videos/[^/]+/(?P<id>[^/?#]+)'
 
     _TEST = {
-        'url': 'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html',
+        'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
         # The md5 checksum changes
         'info_dict': {
             'id': 'qurhIVlJSB6hzkVi229d8g',
             'ext': 'flv',
             'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
-            'description': 'md5:0681e0d30dcdfc6abf34594961d8ea88',
+            'description': 'md5:a8ba0302912d03d246979735c17d2761',
         },
     }
 
     def _real_extract(self, url):
         name = self._match_id(url)
         webpage = self._download_webpage(url, name)
-
-        f4m_url = self._search_regex(
-            r'<source src="(https?://[^"]+\.f4m.*?)"', webpage,
-            'f4m url')
+        video_id = self._search_regex(r'"bmmrId":"(.+?)"', webpage, 'id')
         title = re.sub(': Video$', '', self._og_search_title(webpage))
 
+        embed_info = self._download_json(
+            'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
+        formats = []
+        for stream in embed_info['streams']:
+            if stream["muxing_format"] == "TS":
+                formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
+            else:
+                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+        self._sort_formats(formats)
+
         return {
-            'id': name.split('-')[-1],
+            'id': video_id,
             'title': title,
-            'formats': self._extract_f4m_formats(f4m_url, name),
+            'formats': formats,
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
         }

From e81a47460365738a0add4d4da52a712c0091704f Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Fri, 3 Apr 2015 15:34:49 +0200
Subject: [PATCH 0139/1515] [Gamersyde] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/gamersyde.py | 64 +++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 youtube_dl/extractor/gamersyde.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aae4aae4c..2935d5b33 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -177,6 +177,7 @@ from .gameone import (
     GameOneIE,
     GameOnePlaylistIE,
 )
+from .gamersyde import GamersydeIE
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
new file mode 100644
index 000000000..c40106216
--- /dev/null
+++ b/youtube_dl/extractor/gamersyde.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+import json
+import time
+from .common import InfoExtractor
+
+
+class GamersydeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
+    _TEST = {
+        'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
+        'md5': 'f38d400d32f19724570040d5ce3a505f',
+        'info_dict': {
+            'id': '34371',
+            'ext': 'mp4',
+            'title': 'Bloodborne - Birth of a hero',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _calculateDuration(self, durationString):
+        duration = time.strptime(durationString, "%M minutes %S seconds")
+        return duration.tm_min * 60 + duration.tm_sec
+
+    def _fixJsonSyntax(self, json):
+
+        json = re.sub(r"{\s*(\w)", r'{"\1', json)
+        json = re.sub(r",\s*(\w)", r',"\1', json)
+        json = re.sub(r"(\w): ", r'\1":', json)
+        json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
+        json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
+
+        return json
+
+    def _real_extract(self, url):
+
+        video_id = self._search_regex(r'-(.*?)_[a-z]{2}.html$', url, 'video_id')
+        webpage = self._download_webpage(url, video_id)
+
+        filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
+        filesJson = self._fixJsonSyntax(filesJson)
+
+        data = json.loads(filesJson)
+        playlist = data[0]
+
+        formats = []
+
+        title = re.sub(r"[0-9]+ - ", "", playlist['title'])
+
+        for playlistEntry in playlist['sources']:
+            format = {
+                'url': playlistEntry['file'],
+                'format_id': playlistEntry['label']
+            }
+
+            formats.append(format)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': playlist['image']
+            }

From 185a7e25e7c18b0dff17bdb9ae828616a5ac17d4 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Fri, 3 Apr 2015 20:55:39 +0430
Subject: [PATCH 0140/1515] [RadioJavan] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/radiojavan.py | 71 ++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/radiojavan.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3011b784d..df4a7419a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -391,6 +391,7 @@ from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
+from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
new file mode 100644
index 000000000..de90f9270
--- /dev/null
+++ b/youtube_dl/extractor/radiojavan.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import(
+    parse_duration,
+    str_to_int
+)
+
+class RadioJavanIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
+    _TEST = {
+        'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam',
+        'md5': 'e85208ffa3ca8b83534fca9fe19af95b',
+        'info_dict': {
+            'id': 'chaartaar-ashoobam',
+            'ext': 'mp4',
+            'title': 'Chaartaar - Ashoobam',
+            'description': 'Chaartaar - Ashoobam',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        urls = list()
+        prefix = 'https://media.rdjavan.com/media/music_video/'
+
+        video_url_480 = self._search_regex(
+            r'RJ\.video480p = \'([^\']+)\'', webpage, '480 video url', fatal= False)
+        video_url_720 = self._search_regex(
+            r'RJ\.video720p = \'([^\']+)\'', webpage, '720 video url', fatal= False)
+        video_url_1080 = self._search_regex(
+            r'RJ\.video1080p = \'([^\']+)\'', webpage, '1080 video url', fatal= False)
+
+        if video_url_480:
+            urls.append({'url': prefix + video_url_480, 'format': '480p'})
+        if video_url_720:
+            urls.append({'url': prefix + video_url_720, 'format': '720p'})
+        if video_url_1080:
+            urls.append({'url': prefix + video_url_1080, 'format': '1080p'})
+
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        formats = [{
+            'url': url['url'],
+            'format': url['format']
+        } for url in urls]
+
+        likes = self._search_regex(
+            r'<span class="rating">([\d,]+)\s*likes</span>', webpage, 'Likes Count', fatal=False )
+        likes = likes.replace(',', '')
+        dislikes = self._search_regex(
+            r'<span class="rating">([\d,]+)\s*dislikes</span>', webpage, 'Dislikes Count', fatal=False )
+        dislikes = dislikes.replace(',', '')
+
+        plays = self._search_regex(
+            r'views_publish[">\s]*<span[^>]+class="views">Plays: ([\d,]+)</span>', webpage, 'Play Count', fatal=False )
+        plays = plays.replace(',', '')
+
+        return {
+            'formats': formats,
+            'id': display_id,
+            'title': title,
+            'description': title, # no description provided in RadioJavan
+            'thumbnail': thumbnail,
+            'like_count': str_to_int(likes),
+            'dislike_count': str_to_int(dislikes),
+            'viewCount': str_to_int(plays)
+        }
\ No newline at end of file

From cd341b6e0679c11b3698191615b18dbaaf2b0a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 19:37:35 +0200
Subject: [PATCH 0141/1515] [mixcloud] Fix extraction of like count (reported
 in #5231)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 21aea0c55..84f291558 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -97,7 +97,7 @@ class MixcloudIE(InfoExtractor):
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
         like_count = str_to_int(self._search_regex(
-            r'\bbutton-favorite\b.+m-ajax-toggle-count="([^"]+)"',
+            r'\bbutton-favorite\b[^>]+m-ajax-toggle-count="([^"]+)"',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From 01534bf54f7eb80e4414cba86f1ac6fa608fc6e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Apr 2015 23:42:53 +0600
Subject: [PATCH 0142/1515] [prosiebensat1] Fix bitrate (Closes #5350 closes
 #5351)

---
 youtube_dl/extractor/prosiebensat1.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 385681d06..c46aaada6 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     unified_strdate,
+    int_or_none,
 )
 
 
@@ -266,6 +267,9 @@ class ProSiebenSat1IE(InfoExtractor):
             urls_sources = urls_sources.values()
 
         def fix_bitrate(bitrate):
+            bitrate = int_or_none(bitrate)
+            if not bitrate:
+                return None
             return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
 
         for source in urls_sources:

From 16fa01291bd94703e2258a68bef1491d57f0dabc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Apr 2015 23:44:13 +0600
Subject: [PATCH 0143/1515] [prosiebensat1] Fix test

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index c46aaada6..7cc799664 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -25,7 +25,7 @@ class ProSiebenSat1IE(InfoExtractor):
             'info_dict': {
                 'id': '2104602',
                 'ext': 'mp4',
-                'title': 'Staffel 2, Episode 18 - Jahresrückblick',
+                'title': 'Episode 18 - Staffel 2',
                 'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                 'upload_date': '20131231',
                 'duration': 5845.04,

From ff556f5c09ec8700bb012a58a5e39505b887b774 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 00:30:37 +0600
Subject: [PATCH 0144/1515] Do not encode outtmpl twice (Closes #5288)

---
 youtube_dl/__init__.py | 4 ----
 youtube_dl/options.py  | 3 ++-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 852b2fc3d..1c8b411b7 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -189,10 +189,6 @@ def _real_main(argv=None):
     if opts.allsubtitles and not opts.writeautomaticsub:
         opts.writesubtitles = True
 
-    if sys.version_info < (3,):
-        # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
-        if opts.outtmpl is not None:
-            opts.outtmpl = opts.outtmpl.decode(preferredencoding())
     outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or
                (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or
                (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 35c7e5fb3..8e80e3759 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -13,6 +13,7 @@ from .compat import (
     compat_kwargs,
 )
 from .utils import (
+    preferredencoding,
     write_string,
 )
 from .version import __version__
@@ -797,7 +798,7 @@ def parseOpts(overrideArguments=None):
         # Workaround for Python 2.x, where argv is a byte list
         if sys.version_info < (3,):
             command_line_conf = [
-                a.decode('utf-8', 'replace') for a in command_line_conf]
+                a.decode(preferredencoding(), 'replace') for a in command_line_conf]
 
         if '--ignore-config' in command_line_conf:
             system_conf = []

From 115c281672bd7479f87c48249f6a0186ac7d19cc Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sat, 4 Apr 2015 12:31:48 +0200
Subject: [PATCH 0145/1515] [Gamersyde] Improved robustness, added duration and
 tests

Fix for Json syntax is now less error prone for Json syntax inside of
values. Extractor is now also using native Json handling. Added tests
for several videos that were producing errors in the first place.
---
 youtube_dl/extractor/gamersyde.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
index c40106216..5c68a6891 100644
--- a/youtube_dl/extractor/gamersyde.py
+++ b/youtube_dl/extractor/gamersyde.py
@@ -8,7 +8,6 @@ from .common import InfoExtractor
 
 class GamersydeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
-    _TEST = {
         'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
         'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
@@ -17,6 +16,11 @@ class GamersydeIE(InfoExtractor):
             'title': 'Bloodborne - Birth of a hero',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
+    },
+    {
+        'url': 'http://www.gamersyde.com/hqstream_dark_souls_ii_scholar_of_the_first_sin_gameplay_part_1-34417_en.html',
+        'info_dict': {
+            'ext': 'mp4',
     }
 
     def _calculateDuration(self, durationString):
@@ -27,7 +31,6 @@ class GamersydeIE(InfoExtractor):
 
         json = re.sub(r"{\s*(\w)", r'{"\1', json)
         json = re.sub(r",\s*(\w)", r',"\1', json)
-        json = re.sub(r"(\w): ", r'\1":', json)
         json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
         json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
 
@@ -40,7 +43,6 @@ class GamersydeIE(InfoExtractor):
 
         filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
         filesJson = self._fixJsonSyntax(filesJson)
-
         data = json.loads(filesJson)
         playlist = data[0]
 

From 3d24d997ae1f92686aa7edd0bfeed28353fbfb2e Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sat, 4 Apr 2015 12:42:14 +0200
Subject: [PATCH 0146/1515] Fixed intendation of test cases

Leaded to error on Linux machine
---
 youtube_dl/extractor/gamersyde.py | 45 ++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
index 5c68a6891..cc6fa4037 100644
--- a/youtube_dl/extractor/gamersyde.py
+++ b/youtube_dl/extractor/gamersyde.py
@@ -1,39 +1,62 @@
 # coding: utf-8
 from __future__ import unicode_literals
 import re
-import json
 import time
+
 from .common import InfoExtractor
 
 
 class GamersydeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
+    _TESTS = [{
         'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
         'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
             'id': '34371',
             'ext': 'mp4',
+            'duration': 372,
             'title': 'Bloodborne - Birth of a hero',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    },
-    {
+    }, {
         'url': 'http://www.gamersyde.com/hqstream_dark_souls_ii_scholar_of_the_first_sin_gameplay_part_1-34417_en.html',
+        'md5': '94bd7c3feff3275576cf5cb6c8a3a720',
         'info_dict': {
+            'id': '34417',
             'ext': 'mp4',
+            'duration': 270,
+            'title': 'Dark Souls II: Scholar of the First Sin - Gameplay - Part 1',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.gamersyde.com/hqstream_grand_theft_auto_v_heists_trailer-33786_en.html',
+        'md5': '65e442f5f340d571ece8c80d50700369',
+        'info_dict': {
+            'id': '33786',
+            'ext': 'mp4',
+            'duration': 59,
+            'title': 'Grand Theft Auto V - Heists Trailer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
     }
+    ]
 
     def _calculateDuration(self, durationString):
-        duration = time.strptime(durationString, "%M minutes %S seconds")
+        if (durationString.find("minutes") > -1):
+            duration = time.strptime(durationString, "%M minutes %S seconds")
+        else:
+            duration = time.strptime(durationString, "%S seconds")
         return duration.tm_min * 60 + duration.tm_sec
 
     def _fixJsonSyntax(self, json):
 
-        json = re.sub(r"{\s*(\w)", r'{"\1', json)
-        json = re.sub(r",\s*(\w)", r',"\1', json)
         json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
         json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
-
+        json = json.replace('file: "', '"file": "')
+        json = json.replace('title: "', '"title": "')
+        json = json.replace('label: "', '"label": "')
+        json = json.replace('image: "', '"image": "')
+        json = json.replace('sources: [', '"sources": [')
         return json
 
     def _real_extract(self, url):
@@ -42,13 +65,16 @@ class GamersydeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
-        filesJson = self._fixJsonSyntax(filesJson)
-        data = json.loads(filesJson)
+        data = self._parse_json(filesJson,video_id, transform_source=self._fixJsonSyntax)
+        
         playlist = data[0]
 
         formats = []
 
         title = re.sub(r"[0-9]+ - ", "", playlist['title'])
+        
+        length = self._search_regex(r'(([0-9]{1,2} minutes ){0,1}[0-9]{1,2} seconds)', webpage, 'length')
+        duration = self._calculateDuration(length)
 
         for playlistEntry in playlist['sources']:
             format = {
@@ -62,5 +88,6 @@ class GamersydeIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'duration': duration,
             'thumbnail': playlist['image']
             }

From 7cf97daf77f6419f2b965a199a3fb1e63b8771b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 16:45:41 +0600
Subject: [PATCH 0147/1515] [radiojavan] Simplify and extract upload date

---
 youtube_dl/extractor/radiojavan.py | 75 ++++++++++++++----------------
 1 file changed, 35 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index de90f9270..73ab78d6d 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -1,12 +1,14 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import(
-    parse_duration,
-    str_to_int
+    unified_strdate,
+    str_to_int,
 )
 
+
 class RadioJavanIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
     _TEST = {
@@ -16,56 +18,49 @@ class RadioJavanIE(InfoExtractor):
             'id': 'chaartaar-ashoobam',
             'ext': 'mp4',
             'title': 'Chaartaar - Ashoobam',
-            'description': 'Chaartaar - Ashoobam',
             'thumbnail': 're:^https?://.*\.jpe?g$',
+            'upload_date': '20150215',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
         }
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        urls = list()
-        prefix = 'https://media.rdjavan.com/media/music_video/'
+        video_id = self._match_id(url)
 
-        video_url_480 = self._search_regex(
-            r'RJ\.video480p = \'([^\']+)\'', webpage, '480 video url', fatal= False)
-        video_url_720 = self._search_regex(
-            r'RJ\.video720p = \'([^\']+)\'', webpage, '720 video url', fatal= False)
-        video_url_1080 = self._search_regex(
-            r'RJ\.video1080p = \'([^\']+)\'', webpage, '1080 video url', fatal= False)
+        webpage = self._download_webpage(url, video_id)
 
-        if video_url_480:
-            urls.append({'url': prefix + video_url_480, 'format': '480p'})
-        if video_url_720:
-            urls.append({'url': prefix + video_url_720, 'format': '720p'})
-        if video_url_1080:
-            urls.append({'url': prefix + video_url_1080, 'format': '1080p'})
+        formats = [{
+            'url': 'https://media.rdjavan.com/media/music_video/%s' % video_path,
+            'format_id': '%sp' % height,
+            'height': height,
+        } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        formats = [{
-            'url': url['url'],
-            'format': url['format']
-        } for url in urls]
 
-        likes = self._search_regex(
-            r'<span class="rating">([\d,]+)\s*likes</span>', webpage, 'Likes Count', fatal=False )
-        likes = likes.replace(',', '')
-        dislikes = self._search_regex(
-            r'<span class="rating">([\d,]+)\s*dislikes</span>', webpage, 'Dislikes Count', fatal=False )
-        dislikes = dislikes.replace(',', '')
+        upload_date = unified_strdate(self._search_regex(
+            r'class="date_added">Date added: ([^<]+)<',
+            webpage, 'upload date', fatal=False))
 
-        plays = self._search_regex(
-            r'views_publish[">\s]*<span[^>]+class="views">Plays: ([\d,]+)</span>', webpage, 'Play Count', fatal=False )
-        plays = plays.replace(',', '')
+        view_count = str_to_int(self._search_regex(
+            r'class="views">Plays: ([\d,]+)',
+            webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            r'class="rating">([\d,]+) likes',
+            webpage, 'like count', fatal=False))
+        dislike_count = str_to_int(self._search_regex(
+            r'class="rating">([\d,]+) dislikes',
+            webpage, 'dislike count', fatal=False))
 
         return {
-            'formats': formats,
-            'id': display_id,
+            'id': video_id,
             'title': title,
-            'description': title, # no description provided in RadioJavan
             'thumbnail': thumbnail,
-            'like_count': str_to_int(likes),
-            'dislike_count': str_to_int(dislikes),
-            'viewCount': str_to_int(plays)
-        }
\ No newline at end of file
+            'upload_date': upload_date,
+            'view_count': view_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'formats': formats,
+        }

From 6e617ed0b6b5bb932f928f63c2bda36f5317468d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 16:47:09 +0600
Subject: [PATCH 0148/1515] Credit @mtp1376 for varzesh3 and radiojavan

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 48769320a..cf238176b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -120,3 +120,4 @@ Jeff Buchbinder
 Amish Bhadeshia
 Joram Schrijver
 Will W.
+Mohammad Teimori Pabandi

From e9f65f87496d740fbb61e036c710bf2c174f1cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Apr 2015 13:11:55 +0200
Subject: [PATCH 0149/1515] [rtve] Extract a better quality video

---
 youtube_dl/extractor/rtve.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 13f071077..8d9be1b98 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -10,6 +10,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     float_or_none,
     remove_end,
+    std_headers,
     struct_unpack,
 )
 
@@ -84,13 +85,20 @@ class RTVEALaCartaIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _real_initialize(self):
+        user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
+        manager_info = self._download_json(
+            'http://www.rtve.es/odin/loki/' + user_agent_b64,
+            None, 'Fetching manager info')
+        self._manager = manager_info['manager']
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
+        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):

From ba9e68f40261355ceae5bb87c5707adc7f7beb2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 17:48:55 +0600
Subject: [PATCH 0150/1515] [utils] Drop trailing comma before closing brace

---
 test/test_utils.py  | 6 ++++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index abaf1ab73..4e524aca3 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -470,6 +470,12 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(d['x'], 1)
         self.assertEqual(d['y'], 'a')
 
+        on = js_to_json('["abc", "def",]')
+        self.assertEqual(json.loads(on), ['abc', 'def'])
+
+        on = js_to_json('{"abc": "def",}')
+        self.assertEqual(json.loads(on), {'abc': 'def'})
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 90e0ed9ab..e1761265c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1577,7 +1577,7 @@ def js_to_json(code):
         '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
         [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
-    res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
+    res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
     return res
 
 

From 5c29dbd0c76083eaf596f623fabb612575f71861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 17:53:22 +0600
Subject: [PATCH 0151/1515] [gamersyde] Simplify

---
 youtube_dl/extractor/gamersyde.py | 103 ++++++++++++------------------
 1 file changed, 40 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
index cc6fa4037..d545e01bb 100644
--- a/youtube_dl/extractor/gamersyde.py
+++ b/youtube_dl/extractor/gamersyde.py
@@ -1,14 +1,18 @@
-# coding: utf-8
 from __future__ import unicode_literals
+
 import re
-import time
 
 from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    parse_duration,
+    remove_start,
+)
 
 
 class GamersydeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_(?P<display_id>[\da-z_]+)-(?P<id>\d+)_[a-z]{2}\.html'
+    _TEST = {
         'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
         'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
@@ -18,76 +22,49 @@ class GamersydeIE(InfoExtractor):
             'title': 'Bloodborne - Birth of a hero',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }, {
-        'url': 'http://www.gamersyde.com/hqstream_dark_souls_ii_scholar_of_the_first_sin_gameplay_part_1-34417_en.html',
-        'md5': '94bd7c3feff3275576cf5cb6c8a3a720',
-        'info_dict': {
-            'id': '34417',
-            'ext': 'mp4',
-            'duration': 270,
-            'title': 'Dark Souls II: Scholar of the First Sin - Gameplay - Part 1',
-            'thumbnail': 're:^https?://.*\.jpg$',
-        }
-    }, {
-        'url': 'http://www.gamersyde.com/hqstream_grand_theft_auto_v_heists_trailer-33786_en.html',
-        'md5': '65e442f5f340d571ece8c80d50700369',
-        'info_dict': {
-            'id': '33786',
-            'ext': 'mp4',
-            'duration': 59,
-            'title': 'Grand Theft Auto V - Heists Trailer',
-            'thumbnail': 're:^https?://.*\.jpg$',
-        }
     }
-    ]
-
-    def _calculateDuration(self, durationString):
-        if (durationString.find("minutes") > -1):
-            duration = time.strptime(durationString, "%M minutes %S seconds")
-        else:
-            duration = time.strptime(durationString, "%S seconds")
-        return duration.tm_min * 60 + duration.tm_sec
-
-    def _fixJsonSyntax(self, json):
-
-        json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
-        json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
-        json = json.replace('file: "', '"file": "')
-        json = json.replace('title: "', '"title": "')
-        json = json.replace('label: "', '"label": "')
-        json = json.replace('image: "', '"image": "')
-        json = json.replace('sources: [', '"sources": [')
-        return json
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        video_id = self._search_regex(r'-(.*?)_[a-z]{2}.html$', url, 'video_id')
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
-        data = self._parse_json(filesJson,video_id, transform_source=self._fixJsonSyntax)
-        
-        playlist = data[0]
+        playlist = self._parse_json(
+            self._search_regex(
+                r'(?s)playlist: \[({.+?})\]\s*}\);', webpage, 'files'),
+            display_id, transform_source=js_to_json)
 
         formats = []
-
-        title = re.sub(r"[0-9]+ - ", "", playlist['title'])
-        
-        length = self._search_regex(r'(([0-9]{1,2} minutes ){0,1}[0-9]{1,2} seconds)', webpage, 'length')
-        duration = self._calculateDuration(length)
-
-        for playlistEntry in playlist['sources']:
-            format = {
-                'url': playlistEntry['file'],
-                'format_id': playlistEntry['label']
+        for source in playlist['sources']:
+            video_url = source.get('file')
+            if not video_url:
+                continue
+            format_id = source.get('label')
+            f = {
+                'url': video_url,
+                'format_id': format_id,
             }
+            m = re.search(r'^(?P<height>\d+)[pP](?P<fps>\d+)fps', format_id)
+            if m:
+                f.update({
+                    'height': int(m.group('height')),
+                    'fps': int(m.group('fps')),
+                })
+            formats.append(f)
+        self._sort_formats(formats)
 
-            formats.append(format)
+        title = remove_start(playlist['title'], '%s - ' % video_id)
+        thumbnail = playlist.get('image')
+        duration = parse_duration(self._search_regex(
+            r'Length:</label>([^<]+)<', webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
-            'formats': formats,
+            'thumbnail': thumbnail,
             'duration': duration,
-            'thumbnail': playlist['image']
-            }
+            'formats': formats,
+        }

From 79c21abba7c9902f00ddac83a2af29c36fe0e122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 18:45:46 +0600
Subject: [PATCH 0152/1515] [utils] Add one more template to unified_strdate

---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e1761265c..be3f62da7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -759,6 +759,7 @@ def unified_strdate(date_str, day_first=True):
     ]
     if day_first:
         format_expressions.extend([
+            '%d-%m-%Y',
             '%d.%m.%Y',
             '%d/%m/%Y',
             '%d/%m/%y',
@@ -766,6 +767,7 @@ def unified_strdate(date_str, day_first=True):
         ])
     else:
         format_expressions.extend([
+            '%m-%d-%Y',
             '%m.%d.%Y',
             '%m/%d/%Y',
             '%m/%d/%y',

From 15ac8413c78b991f2e99b6bdc538bc8c5ae8e8a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:08:48 +0600
Subject: [PATCH 0153/1515] [utils] Avoid treating `*-%Y` date template as UTC
 offset

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index be3f62da7..52f0dd09a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -730,7 +730,8 @@ def unified_strdate(date_str, day_first=True):
     # Replace commas
     date_str = date_str.replace(',', ' ')
     # %z (UTC offset) is only supported in python>=3.2
-    date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
+    if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
+        date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
     # Remove AM/PM + timezone
     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 

From 8cf70de428c3fef910ba966fb56d39478226acc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:11:01 +0600
Subject: [PATCH 0154/1515] [test_utils] Add test for unified_strdate

---
 test/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4e524aca3..2e3a6480c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -227,6 +227,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
             '20150202')
+        self.assertEqual(unified_strdate('25-09-2014'), '20140925')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>

From 7c39a65543b809b681434246b84710349f5837aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:13:37 +0600
Subject: [PATCH 0155/1515] [pornovoisines] Simplify

---
 youtube_dl/extractor/pornovoisines.py | 111 ++++++++++++--------------
 1 file changed, 53 insertions(+), 58 deletions(-)

diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index efbb6a818..9688ed948 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -2,19 +2,23 @@
 from __future__ import unicode_literals
 
 import re
-import datetime
 import random
 
-from ..compat import compat_urllib_parse
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    unified_strdate,
+)
+
 
 class PornoVoisinesIE(InfoExtractor):
-    _VALID_URL = r'^((?:http://)?(?:www\.)?pornovoisines.com)/showvideo/(\d+)/([^/]+)'
+    _VALID_URL = r'http://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
 
-    VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
+    _VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
         '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
 
-    SERVER_NUMBERS = (1, 2)
+    _SERVER_NUMBERS = (1, 2)
 
     _TEST = {
         'url': 'http://www.pornovoisines.com/showvideo/1285/recherche-appartement/',
@@ -23,79 +27,70 @@ class PornoVoisinesIE(InfoExtractor):
             'id': '1285',
             'display_id': 'recherche-appartement',
             'ext': 'mp4',
-            'title': "Recherche appartement",
+            'title': 'Recherche appartement',
+            'description': 'md5:819ea0b785e2a04667a1a01cdc89594e',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'upload_date': '20140925',
-            'view_count': int,
             'duration': 120,
-            'categories': ["Débutante", "Scénario", "Sodomie"],
-            'description': 're:^Pour la .+ original...$',
-            'thumbnail': 're:^http://',
-            'uploader': "JMTV",
+            'view_count': int,
             'average_rating': float,
-            'comment_count': int,
+            'categories': ['Débutante', 'Scénario', 'Sodomie'],
             'age_limit': 18,
         }
     }
 
     @classmethod
-    def build_video_url(cls, id):
-        server_nr = random.choice(cls.SERVER_NUMBERS)
-        return cls.VIDEO_URL_TEMPLATE % (server_nr, id)
-
-    @staticmethod
-    def parse_upload_date(str):
-        return datetime.datetime.strptime(str, "%d-%m-%Y").strftime("%Y%m%d")
-
-    @staticmethod
-    def parse_categories(str):
-        return map(lambda s: s.strip(), str.split(','))
+    def build_video_url(cls, num):
+        return cls._VIDEO_URL_TEMPLATE % (random.choice(cls._SERVER_NUMBERS), num)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        url_prefix = mobj.group(1)
-        id = mobj.group(2)
-        display_id = mobj.group(3)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title',
-            flags=re.DOTALL)
-        url = self.build_video_url(id)
-        upload_date = self.parse_upload_date(
-            self._search_regex(r'Publié le (\d\d-\d\d-\d{4})', webpage,
-            'upload date'))
-        view_count = int(self._search_regex(r'(\d+) vues', webpage, 'view count'))
-        duration = int(self._search_regex('Durée (\d+)', webpage, 'duration'))
-        categories = self.parse_categories(self._html_search_regex(
-            r'<li class="categorie">(.+?)</li>', webpage, "categories",
-            flags=re.DOTALL))
+        video_url = self.build_video_url(video_id)
+
+        title = self._html_search_regex(
+            r'<h1>(.+?)</h1>', webpage, 'title', flags=re.DOTALL)
         description = self._html_search_regex(
-            r'<article id="descriptif">(.+?)</article>', webpage, "description",
-            flags=re.DOTALL)
-        thumbnail = url_prefix + self._html_search_regex(re.compile(
-            '<div id="mediaspace' + id + '">.*?<img src="(.+?)"', re.DOTALL),
-            webpage, "thumbnail")
-        uploader = re.sub(r' *\| *$', '',
-            self._html_search_regex(r'<li class="auteur">(.+?)</li>', webpage,
-            "uploader", flags=re.DOTALL))
-        average_rating = float(self._search_regex(r'Note : (\d+,\d+)',
-            webpage, "average rating").replace(',', '.'))
-        comment_count = int(self._search_regex(r'\((\d+)\)', webpage,
-            "comment count"))
+            r'<article id="descriptif">(.+?)</article>',
+            webpage, "description", fatal=False, flags=re.DOTALL)
+
+        thumbnail = self._search_regex(
+            r'<div id="mediaspace%s">\s*<img src="/?([^"]+)"' % video_id,
+            webpage, 'thumbnail', fatal=False)
+        if thumbnail:
+            thumbnail = 'http://www.pornovoisines.com/%s' % thumbnail
+
+        upload_date = unified_strdate(self._search_regex(
+            r'Publié le ([\d-]+)', webpage, 'upload date', fatal=False))
+        duration = int_or_none(self._search_regex(
+            'Durée (\d+)', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._search_regex(
+            r'(\d+) vues', webpage, 'view count', fatal=False))
+        average_rating = self._search_regex(
+            r'Note : (\d+,\d+)', webpage, 'average rating', fatal=False)
+        if average_rating:
+            average_rating = float_or_none(average_rating.replace(',', '.'))
+
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', fatal=False)
+        if categories:
+            categories = [category.strip() for category in categories.split(',')]
 
         return {
-            'id': id,
+            'id': video_id,
             'display_id': display_id,
-            'url': url,
+            'url': video_url,
             'title': title,
-            'upload_date': upload_date,
-            'view_count': view_count,
-            'duration': duration,
-            'categories': categories,
             'description': description,
             'thumbnail': thumbnail,
-            'uploader': uploader,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
             'average_rating': average_rating,
-            'comment_count': comment_count,
+            'categories': categories,
             'age_limit': 18,
         }

From 424266abb10eafe6c57c5d391a947fa190a365e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:16:18 +0600
Subject: [PATCH 0156/1515] Credit @Roman2K for pornovoisines (#5264)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cf238176b..9c65dc1d4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -121,3 +121,4 @@ Amish Bhadeshia
 Joram Schrijver
 Will W.
 Mohammad Teimori Pabandi
+Roman Le Négrate

From ff02a228e35ab11c9cfa6e0d000b7fd6de52a0c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:21:50 +0600
Subject: [PATCH 0157/1515] [test_execution] Fix test under python 2 @ windows

---
 test/test_execution.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/test_execution.py b/test/test_execution.py
index f31e51558..620db080e 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -8,6 +8,9 @@ import unittest
 import sys
 import os
 import subprocess
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.utils import encodeArgument
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -31,7 +34,7 @@ class TestExecution(unittest.TestCase):
 
     def test_cmdline_umlauts(self):
         p = subprocess.Popen(
-            [sys.executable, 'youtube_dl/__main__.py', 'ä', '--version'],
+            [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'],
             cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
         _, stderr = p.communicate()
         self.assertFalse(stderr)

From 4e8cc1e973da2656c46c5df84d4e85c5d78836ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:24:37 +0600
Subject: [PATCH 0158/1515] [radiojavan] Fix height

---
 youtube_dl/extractor/radiojavan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index 73ab78d6d..c9eda9b53 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -34,7 +34,7 @@ class RadioJavanIE(InfoExtractor):
         formats = [{
             'url': 'https://media.rdjavan.com/media/music_video/%s' % video_path,
             'format_id': '%sp' % height,
-            'height': height,
+            'height': int(height),
         } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
 
         title = self._og_search_title(webpage)

From 8fb2e5a4f5b9604f93964f9b6ae7062830e3bab2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:25:08 +0600
Subject: [PATCH 0159/1515] [radiojavan] Sort formats

---
 youtube_dl/extractor/radiojavan.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index c9eda9b53..884c28420 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -36,6 +36,7 @@ class RadioJavanIE(InfoExtractor):
             'format_id': '%sp' % height,
             'height': int(height),
         } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
+        self._sort_formats(formats)
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)

From 4a3cdf81af9c22c45912b0b4c5845531d52d3a0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:00:23 +0600
Subject: [PATCH 0160/1515] [options] Restore some strings

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9bded4521..d861ac458 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -359,7 +359,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--max-quality',
         action='store', dest='format_limit', metavar='FORMAT',
-        help='Specify highest quality format to download')
+        help='Highest quality format to download')
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
@@ -399,7 +399,7 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--sub-format',
         action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
-        help='Specify subtitle format preference, for example: "srt" or "ass/srt/best"')
+        help='Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"')
     subtitles.add_option(
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',

From f01855813b364dbd3e0c7fecacda84410d2780bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:01:24 +0600
Subject: [PATCH 0161/1515] [options] extractor is lowercase

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d861ac458..ed2216d40 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -606,7 +606,7 @@ def parseOpts(overrideArguments=None):
               '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
               '%(format_id)s for the unique id of the format (like YouTube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
-              '%(extractor)s for the provider (YouTube, metacafe, etc), '
+              '%(extractor)s for the provider (youtube, metacafe, etc), '
               '%(id)s for the video id, '
               '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
               '%(playlist_index)s for the position in the playlist. '

From 6b70a4eb7d4bcbe6812f78876b4aa9aa44a58fef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:02:29 +0600
Subject: [PATCH 0162/1515] [options] `Number` is a verb here

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ed2216d40..2097a9436 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -626,7 +626,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number of downloaded files starting from 00000')
+        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From 1a48181a9ff872e4b8428603f70851c386a2790d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:09:11 +0600
Subject: [PATCH 0163/1515] [options] Fix load info help string

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2097a9436..5720fb424 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -670,7 +670,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
-        help='Specify JSON file containing the video information (created with the "--write-json" option)')
+        help='JSON file containing the video information (created with the "--write-info-json" option)')
     filesystem.add_option(
         '--cookies',
         dest='cookiefile', metavar='FILE',

From 1a68d39211cab61994c8717cce296b0baae8095a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:15:59 +0600
Subject: [PATCH 0164/1515] [aftonbladet] Fix extraction

---
 youtube_dl/extractor/aftonbladet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index 8442019ea..4675585ca 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'^http://tv\.aftonbladet\.se/webbtv.+?(?P<video_id>article[0-9]+)\.ab(?:$|[?#])'
+    _VALID_URL = r'http://tv\.aftonbladet\.se/webbtv.+?(?P<id>article[0-9]+)\.ab(?:$|[?#])'
     _TEST = {
         'url': 'http://tv.aftonbladet.se/webbtv/nyheter/vetenskap/rymden/article36015.ab',
         'info_dict': {

From 8e1f93747338d64f6855c0f7f9467714bf56db93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:19:34 +0600
Subject: [PATCH 0165/1515] [aftonbladet] Modernize

---
 youtube_dl/extractor/aftonbladet.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index 4675585ca..a117502bc 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
@@ -43,9 +44,9 @@ class AftonbladetIE(InfoExtractor):
             formats.append({
                 'url': 'http://%s:%d/%s/%s' % (p['address'], p['port'], p['path'], p['filename']),
                 'ext': 'mp4',
-                'width': fmt['width'],
-                'height': fmt['height'],
-                'tbr': fmt['bitrate'],
+                'width': int_or_none(fmt.get('width')),
+                'height': int_or_none(fmt.get('height')),
+                'tbr': int_or_none(fmt.get('bitrate')),
                 'protocol': 'http',
             })
         self._sort_formats(formats)
@@ -54,9 +55,9 @@ class AftonbladetIE(InfoExtractor):
             'id': video_id,
             'title': internal_meta_json['title'],
             'formats': formats,
-            'thumbnail': internal_meta_json['imageUrl'],
-            'description': internal_meta_json['shortPreamble'],
-            'timestamp': internal_meta_json['timePublished'],
-            'duration': internal_meta_json['duration'],
-            'view_count': internal_meta_json['views'],
+            'thumbnail': internal_meta_json.get('imageUrl'),
+            'description': internal_meta_json.get('shortPreamble'),
+            'timestamp': int_or_none(internal_meta_json.get('timePublished')),
+            'duration': int_or_none(internal_meta_json.get('duration')),
+            'view_count': int_or_none(internal_meta_json.get('views')),
         }

From ed676e8c0ab087acb8e5e26a2a8d94a47fe10c33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:27:25 +0600
Subject: [PATCH 0166/1515] [bliptv] Check format URLs

Some formats are now 404
---
 youtube_dl/extractor/bliptv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 8c7ba4b91..b632ce967 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -172,6 +172,7 @@ class BlipTVIE(InfoExtractor):
                     'width': int_or_none(media_content.get('width')),
                     'height': int_or_none(media_content.get('height')),
                 })
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         subtitles = self.extract_subtitles(video_id, subtitles_urls)

From 184a1974414bc91c5804251a33b8dd5cba1f75d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:43:34 +0600
Subject: [PATCH 0167/1515] [culturebox] Check for unavailable videos

---
 youtube_dl/extractor/francetv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 170d68075..20acc96bd 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -275,7 +275,12 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
+
         webpage = self._download_webpage(url, name)
+
+        if ">Ce live n'est plus disponible en replay<" in webpage:
+            raise ExtractorError('Video %s is not available' % name, expected=True)
+
         video_id, catalogue = self._search_regex(
             r'"http://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
 

From aed2d4b31e331422fefa304ab3fa49c050ea13e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:50:13 +0600
Subject: [PATCH 0168/1515] [culturebox] Replace test

---
 youtube_dl/extractor/francetv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 20acc96bd..55b8e9d9e 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -260,15 +260,18 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'
 
     _TEST = {
-        'url': 'http://culturebox.francetvinfo.fr/festivals/dans-les-jardins-de-william-christie/dans-les-jardins-de-william-christie-le-camus-162553',
-        'md5': '5ad6dec1ffb2a3fbcb20cc4b744be8d6',
+        'url': 'http://culturebox.francetvinfo.fr/live/musique/musique-classique/le-livre-vermeil-de-montserrat-a-la-cathedrale-delne-214511',
         'info_dict': {
-            'id': 'EV_22853',
-            'ext': 'flv',
-            'title': 'Dans les jardins de William Christie - Le Camus',
-            'description': 'md5:4710c82315c40f0c865ca8b9a68b5299',
-            'upload_date': '20140829',
-            'timestamp': 1409317200,
+            'id': 'EV_50111',
+            'ext': 'mp4',
+            'title': "Le Livre Vermeil de Montserrat à la Cathédrale d'Elne",
+            'description': 'md5:f8a4ad202e8fe533e2c493cc12e739d9',
+            'upload_date': '20150320',
+            'timestamp': 1426892400,
+            'duration': 2760.9,
+        },
+        'params': {
+            'skip_download': True,
         },
     }
 

From f05d0e73c6d38d86393f8f552fbfcdda80f37607 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:52:25 +0600
Subject: [PATCH 0169/1515] [francetv] Fix duration

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 55b8e9d9e..0d92ef9c4 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -14,6 +14,7 @@ from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
+    float_or_none,
     parse_duration,
 )
 
@@ -86,7 +87,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'title': info['titre'],
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
-            'duration': parse_duration(info['duree']),
+            'duration': float_or_none(info.get('real_duration'), 1000) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
         }

From bc03228ab52672666b79c9fadfbf886f8d8bf5d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:02:04 +0600
Subject: [PATCH 0170/1515] [francetv] Improve formats extraction

---
 youtube_dl/extractor/francetv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 0d92ef9c4..fd3e7aa7b 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -16,6 +16,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_duration,
+    determine_ext,
 )
 
 
@@ -51,7 +52,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             if not video_url:
                 continue
             format_id = video['format']
-            if video_url.endswith('.f4m'):
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
                 if georestricted:
                     # See https://github.com/rg3/youtube-dl/issues/3963
                     # m3u8 urls work fine
@@ -61,12 +63,9 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     'http://hdfauth.francetv.fr/esi/urltokengen2.html?url=%s' % video_url_parsed.path,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
-                    f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
-                    for f4m_format in f4m_formats:
-                        f4m_format['preference'] = 1
-                    formats.extend(f4m_formats)
-            elif video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
+                    formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
             elif video_url.startswith('rtmp'):
                 formats.append({
                     'url': video_url,

From e21a55abcc502abac559027551751ff84d215077 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:05:25 +0600
Subject: [PATCH 0171/1515] [extractor/common] Remove f4m section

It's now provided by `f4m_id`
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e5245ec3f..530c449c1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -822,7 +822,7 @@ class InfoExtractor(object):
                                 (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
-                'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
+                'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,

From ac651e974ef8da74df35e5ac5464cc03e35bb2d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:06:16 +0600
Subject: [PATCH 0172/1515] [culturebox] Fix test

---
 youtube_dl/extractor/francetv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index fd3e7aa7b..edf555b29 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -261,18 +261,16 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
 
     _TEST = {
         'url': 'http://culturebox.francetvinfo.fr/live/musique/musique-classique/le-livre-vermeil-de-montserrat-a-la-cathedrale-delne-214511',
+        'md5': '9b88dc156781c4dbebd4c3e066e0b1d6',
         'info_dict': {
             'id': 'EV_50111',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': "Le Livre Vermeil de Montserrat à la Cathédrale d'Elne",
             'description': 'md5:f8a4ad202e8fe533e2c493cc12e739d9',
             'upload_date': '20150320',
             'timestamp': 1426892400,
             'duration': 2760.9,
         },
-        'params': {
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):

From aff84bec07fc1919591827543845790074b0194f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:17:09 +0600
Subject: [PATCH 0173/1515] [drtv] Check for unavailable videos

---
 youtube_dl/extractor/drtv.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 8257e35a4..bd1109549 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -26,6 +26,10 @@ class DRTVIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        if '>Programmet er ikke længere tilgængeligt' in webpage:
+            raise ExtractorError(
+                'Video %s is not available' % video_id, expected=True)
+
         video_id = self._search_regex(
             r'data-(?:material-identifier|episode-slug)="([^"]+)"',
             webpage, 'video id')

From 7d2546397209deab14a0ebad6c933ed97e73fe41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:19:28 +0600
Subject: [PATCH 0174/1515] [drtv] Update test

---
 youtube_dl/extractor/drtv.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index bd1109549..f25ab319e 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor, ExtractorError
@@ -8,16 +9,16 @@ class DRTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
-        'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',
-        'md5': '4a7e1dd65cdb2643500a3f753c942f25',
+        'url': 'https://www.dr.dk/tv/se/boern/ultra/panisk-paske/panisk-paske-5',
+        'md5': 'dc515a9ab50577fa14cc4e4b0265168f',
         'info_dict': {
-            'id': 'partiets-mand-7-8',
+            'id': 'panisk-paske-5',
             'ext': 'mp4',
-            'title': 'Partiets mand (7:8)',
-            'description': 'md5:a684b90a8f9336cd4aab94b7647d7862',
-            'timestamp': 1403047940,
-            'upload_date': '20140617',
-            'duration': 1299.040,
+            'title': 'Panisk Påske (5)',
+            'description': 'md5:ca14173c5ab24cd26b0fcc074dff391c',
+            'timestamp': 1426984612,
+            'upload_date': '20150322',
+            'duration': 1455,
         },
     }
 

From 218d6bcc05bd84d8f69a7b764702dc24acb2f761 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:28:47 +0600
Subject: [PATCH 0175/1515] [dreisat] Capture status errors

---
 youtube_dl/extractor/dreisat.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 69ca75423..b88460a23 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+)
 
 
 class DreiSatIE(InfoExtractor):
@@ -28,6 +31,15 @@ class DreiSatIE(InfoExtractor):
         details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         details_doc = self._download_xml(details_url, video_id, 'Downloading video details')
 
+        status_code = details_doc.find('./status/statuscode')
+        if status_code is not None and status_code.text != 'ok':
+            code = status_code.text
+            if code == 'notVisibleAnymore':
+                message = 'Video %s is not available' % video_id
+            else:
+                message = '%s returned error: %s' % (self.IE_NAME, code)
+            raise ExtractorError(message, expected=True)
+
         thumbnail_els = details_doc.findall('.//teaserimage')
         thumbnails = [{
             'width': int(te.attrib['key'].partition('x')[0]),

From a319c33d8b8c4a7ac1d2f8dd739508b041d960b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:30:38 +0600
Subject: [PATCH 0176/1515] [dreisat] Update test

---
 youtube_dl/extractor/dreisat.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index b88460a23..05bb22ddf 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -13,15 +13,15 @@ class DreiSatIE(InfoExtractor):
     IE_NAME = '3sat'
     _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TEST = {
-        'url': 'http://www.3sat.de/mediathek/index.php?obj=36983',
-        'md5': '9dcfe344732808dbfcc901537973c922',
+        'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
+        'md5': 'be37228896d30a88f315b638900a026e',
         'info_dict': {
-            'id': '36983',
+            'id': '45918',
             'ext': 'mp4',
-            'title': 'Kaffeeland Schweiz',
-            'description': 'md5:cc4424b18b75ae9948b13929a0814033',
+            'title': 'Waidmannsheil',
+            'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
             'uploader': '3sat',
-            'upload_date': '20130622'
+            'upload_date': '20140913'
         }
     }
 

From fefc9d121d32321d3609e131e488c443d7af962a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:33:07 +0600
Subject: [PATCH 0177/1515] [dump] Fix title extraction

---
 youtube_dl/extractor/dump.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dump.py b/youtube_dl/extractor/dump.py
index 6b651778a..ff78d4fd2 100644
--- a/youtube_dl/extractor/dump.py
+++ b/youtube_dl/extractor/dump.py
@@ -28,12 +28,12 @@ class DumpIE(InfoExtractor):
         video_url = self._search_regex(
             r's1.addVariable\("file",\s*"([^"]+)"', webpage, 'video URL')
 
-        thumb = self._og_search_thumbnail(webpage)
-        title = self._search_regex(r'<b>([^"]+)</b>', webpage, 'title')
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
 
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
-            'thumbnail': thumb,
+            'thumbnail': thumbnail,
         }

From f67dcc09f5e2f68032e379133e53b07e58c544dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:36:45 +0600
Subject: [PATCH 0178/1515] [eagleplatform] Skip georestricted test

---
 youtube_dl/extractor/eagleplatform.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 7173371ee..688dfc2f7 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -45,6 +45,7 @@ class EaglePlatformIE(InfoExtractor):
             'duration': 216,
             'view_count': int,
         },
+        'skip': 'Georestricted',
     }]
 
     def _handle_error(self, response):

From 27fe5e347350484009e79251ec7ef97484219481 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Apr 2015 00:00:04 +0600
Subject: [PATCH 0179/1515] [ellentv] Make video url extraction fatal

---
 youtube_dl/extractor/ellentv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index fc92ff825..3a7962144 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -40,14 +40,15 @@ class EllenTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        video_url = self._html_search_meta('VideoURL', webpage, 'url')
+
+        video_url = self._html_search_meta('VideoURL', webpage, 'url', fatal=True)
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'pageName\s*=\s*"([^"]+)"', webpage, 'title')
         description = self._html_search_meta(
             'description', webpage, 'description') or self._og_search_description(webpage)
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
-            webpage, 'timestamp'))
+            webpage, 'timestamp', fatal=False))
 
         return {
             'id': video_id,

From 0de9312a7e7c15262ce4a59249d3c4294989757c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Apr 2015 00:01:55 +0600
Subject: [PATCH 0180/1515] [ellentv] Replace test

---
 youtube_dl/extractor/ellentv.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 3a7962144..5154bbd7f 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -13,15 +13,15 @@ from ..utils import (
 class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
     _TESTS = [{
-        'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
-        'md5': 'e4af06f3bf0d5f471921a18db5764642',
+        'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
+        'md5': '8e3c576bf2e9bfff4d76565f56f94c9c',
         'info_dict': {
-            'id': '0-7jqrsr18',
+            'id': '0-ipq1gsai',
             'ext': 'mp4',
-            'title': 'What\'s Wrong with These Photos? A Whole Lot',
-            'description': 'md5:35f152dc66b587cf13e6d2cf4fa467f6',
-            'timestamp': 1406876400,
-            'upload_date': '20140801',
+            'title': 'Fast Fingers of Fate',
+            'description': 'md5:686114ced0a032926935e9015ee794ac',
+            'timestamp': 1428033600,
+            'upload_date': '20150403',
         }
     }, {
         'url': 'http://ellentube.com/videos/0-dvzmabd5/',

From 3a9fadd6dfc127ed0707b218b11ac10c654af1e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 5 Apr 2015 22:29:06 +0800
Subject: [PATCH 0181/1515] [youtube] Enhance url_encoded_fmt_stream_map
 checking (fix #5361)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5488101e1..f7f701cc5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -855,7 +855,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 args = ytplayer_config['args']
                 # Convert to the same format returned by compat_parse_qs
                 video_info = dict((k, [v]) for k, v in args.items())
-                if 'url_encoded_fmt_stream_map' not in args:
+                if ('url_encoded_fmt_stream_map' not in args or
+                        args['url_encoded_fmt_stream_map'] == ''):
                     raise ValueError('No stream_map present')  # caught below
             except ValueError:
                 # We fallback to the get_video_info pages (used by the embed page)

From 06b491eb7b9459f92484f83973c17d46dba59f1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Apr 2015 00:35:55 +0600
Subject: [PATCH 0182/1515] [youtube] Add test for #5361

---
 youtube_dl/extractor/youtube.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f7f701cc5..198fe84ef 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -495,7 +495,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': '孫艾倫',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
             },
-        }
+        },
+        # url_encoded_fmt_stream_map is empty string
+        {
+            'url': 'qEJwOuvDf7I',
+            'info_dict': {
+                'id': 'qEJwOuvDf7I',
+                'ext': 'mp4',
+                'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
+                'description': '',
+                'upload_date': '20150404',
+                'uploader_id': 'spbelect',
+                'uploader': 'Наблюдатели Петербурга',
+            },
+            'params': {
+                'skip_download': 'requires avconv',
+            }
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From e40bd5f06ba397f6a3f9f21ab1df01c120d90eb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Apr 2015 00:45:57 +0600
Subject: [PATCH 0183/1515] [youtube] Simplify url_encoded_fmt_stream_map check

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 198fe84ef..79ce39aa4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -871,8 +871,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 args = ytplayer_config['args']
                 # Convert to the same format returned by compat_parse_qs
                 video_info = dict((k, [v]) for k, v in args.items())
-                if ('url_encoded_fmt_stream_map' not in args or
-                        args['url_encoded_fmt_stream_map'] == ''):
+                if not args.get('url_encoded_fmt_stream_map'):
                     raise ValueError('No stream_map present')  # caught below
             except ValueError:
                 # We fallback to the get_video_info pages (used by the embed page)

From 64102296818f94d3814a8183daa5d92cbdd952fd Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 12:50:21 -0700
Subject: [PATCH 0184/1515] [SpankBang] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/spankbang.py | 38 +++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 youtube_dl/extractor/spankbang.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0f7d44616..e6fdf1297 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -471,6 +471,7 @@ from .southpark import (
     SouthparkDeIE,
 )
 from .space import SpaceIE
+from .spankbang import SpankBangIE
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
new file mode 100644
index 000000000..8e845ef26
--- /dev/null
+++ b/youtube_dl/extractor/spankbang.py
@@ -0,0 +1,38 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+import re
+
+class SpankBangIE(InfoExtractor):
+    """Extractor for http://spankbang.com"""
+    
+    _VALID_URL = r"https?://(?:www\.)?spankbang\.com/(?P<id>\w+)/video/.*"
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r"<h1>(?:<img.+?>)?(.*?)</h1>", webpage, "title")
+        
+        stream_key = self._html_search_regex(r"""var\s+stream_key\s*[=]\s*['"](.+?)['"]\s*;""", webpage, "stream_key")
+        
+        qualities = re.findall(r"<span.+?>([0-9]+p).*?</span>", webpage)
+        
+        formats = []
+        for q in sorted(qualities):
+            formats.append({
+                "format_id": q,
+                "format": q,
+                "ext": "mp4",
+                "url": "http://spankbang.com/_{}/{}/title/{}__mp4".format(video_id, stream_key, q)
+            })
+
+        return {
+            "id": video_id,
+            "title": title,
+            "description": self._og_search_description(webpage),
+            "formats": formats
+        }
+
+# vim: tabstop=4 expandtab

From 2e7daef50220ee90e8a2e2b979600f8bd4a3e40e Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 13:43:21 -0700
Subject: [PATCH 0185/1515] [SpankBang] Use python2.6 compatible string
 formatting spec

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 8e845ef26..61fd64d17 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -25,7 +25,7 @@ class SpankBangIE(InfoExtractor):
                 "format_id": q,
                 "format": q,
                 "ext": "mp4",
-                "url": "http://spankbang.com/_{}/{}/title/{}__mp4".format(video_id, stream_key, q)
+                "url": "http://spankbang.com/_{0}/{1}/title/{2}__mp4".format(video_id, stream_key, q)
             })
 
         return {

From 5c1d459ae91d2681be88023e9056dcae3f48a70a Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 13:57:59 -0700
Subject: [PATCH 0186/1515] [SpankBang] Add test

---
 youtube_dl/extractor/spankbang.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 61fd64d17..2e20a5ad5 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -8,6 +8,20 @@ class SpankBangIE(InfoExtractor):
     """Extractor for http://spankbang.com"""
     
     _VALID_URL = r"https?://(?:www\.)?spankbang\.com/(?P<id>\w+)/video/.*"
+    
+    _TEST = {
+        "url": "http://spankbang.com/3vvn/video/fantasy+solo",
+        "md5": "1cc433e1d6aa14bc376535b8679302f7",
+        "info_dict": {
+            "id": "3vvn",
+            "title": "fantasy solo",
+            "description": "Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.",
+            "format": "720p",
+            "format_id": "720p",
+            "ext": "mp4",
+            "url": "re:http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c7ac5dce8c692f82f10363e40a7085ac53113bc8 Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 14:02:05 -0700
Subject: [PATCH 0187/1515] [SpankBang] Remove regexp type prefix from _TEST
 url.

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 2e20a5ad5..d0b5ba278 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -19,7 +19,7 @@ class SpankBangIE(InfoExtractor):
             "format": "720p",
             "format_id": "720p",
             "ext": "mp4",
-            "url": "re:http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
+            "url": "http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
         }
     }
 

From d2272fcf6e2796583b516f4786733577459dec43 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 6 Apr 2015 09:54:19 +0200
Subject: [PATCH 0188/1515] crooksandliars.com extractor

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/crooksandliars.py | 71 ++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/crooksandliars.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7eb9b4fbb..dc272af82 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,6 +90,7 @@ from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
+from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
new file mode 100644
index 000000000..afccca354
--- /dev/null
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    mimetype2ext,
+)
+
+
+class CrooksAndLiarsIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+        'info_dict': {
+            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'timestamp': 1428207000,
+            'thumbnail': '//crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
+            'uploader': "Heather",
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
+
+        formats = []
+        for item in manifest['flavors']:
+            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
+                continue
+            formats.append({
+                'format_id': item['type'],
+                'ext': mimetype2ext(item['mime']),
+                'url': item['url'],
+            })
+
+        # XXX: manifest['url']?
+        return {
+            'url': url,
+            'id': video_id,
+            'uploader': manifest['author'],
+            'title': manifest['title'],
+            'description': manifest['description'],
+            'thumbnail': manifest['poster'],
+            'duration': manifest['duration'],
+            'timestamp': int(manifest['created']),
+            'formats': formats,
+        }
+
+class CrooksAndLiarsArticleIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
+
+    _TESTS = [{
+        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+        return {
+            '_type': 'url',
+            'url': player_url
+        }

From 6e53c91608d1c43a9fe1614f13a15db74e877a91 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 6 Apr 2015 10:12:43 +0200
Subject: [PATCH 0189/1515] [crooksandliars] resolve protocol-relative URLs

---
 youtube_dl/extractor/crooksandliars.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index afccca354..cee0603f4 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -18,7 +18,7 @@ class CrooksAndLiarsIE(InfoExtractor):
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
             'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
             'timestamp': 1428207000,
-            'thumbnail': '//crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
+            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
             'uploader': "Heather",
         }
     }]
@@ -46,7 +46,7 @@ class CrooksAndLiarsIE(InfoExtractor):
             'uploader': manifest['author'],
             'title': manifest['title'],
             'description': manifest['description'],
-            'thumbnail': manifest['poster'],
+            'thumbnail': self._proto_relative_url(manifest['poster']),
             'duration': manifest['duration'],
             'timestamp': int(manifest['created']),
             'formats': formats,

From d97aae75724fc301243a00c5a71ac93b235d62fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Apr 2015 21:24:17 +0600
Subject: [PATCH 0190/1515] [spankbang] Improve and simplify

---
 youtube_dl/extractor/spankbang.py | 80 +++++++++++++++++--------------
 1 file changed, 44 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index d0b5ba278..7f060b15b 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -1,25 +1,23 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import re
 
+from .common import InfoExtractor
+
+
 class SpankBangIE(InfoExtractor):
-    """Extractor for http://spankbang.com"""
-    
-    _VALID_URL = r"https?://(?:www\.)?spankbang\.com/(?P<id>\w+)/video/.*"
-    
+    _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
     _TEST = {
-        "url": "http://spankbang.com/3vvn/video/fantasy+solo",
-        "md5": "1cc433e1d6aa14bc376535b8679302f7",
-        "info_dict": {
-            "id": "3vvn",
-            "title": "fantasy solo",
-            "description": "Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.",
-            "format": "720p",
-            "format_id": "720p",
-            "ext": "mp4",
-            "url": "http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
+        'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
+        'md5': '1cc433e1d6aa14bc376535b8679302f7',
+        'info_dict': {
+            'id': '3vvn',
+            'ext': 'mp4',
+            'title': 'fantasy solo',
+            'description': 'dillion harper masturbates on a bed',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'silly2587',
+            'age_limit': 18,
         }
     }
 
@@ -27,26 +25,36 @@ class SpankBangIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r"<h1>(?:<img.+?>)?(.*?)</h1>", webpage, "title")
-        
-        stream_key = self._html_search_regex(r"""var\s+stream_key\s*[=]\s*['"](.+?)['"]\s*;""", webpage, "stream_key")
-        
-        qualities = re.findall(r"<span.+?>([0-9]+p).*?</span>", webpage)
-        
-        formats = []
-        for q in sorted(qualities):
-            formats.append({
-                "format_id": q,
-                "format": q,
-                "ext": "mp4",
-                "url": "http://spankbang.com/_{0}/{1}/title/{2}__mp4".format(video_id, stream_key, q)
-            })
+        stream_key = self._html_search_regex(
+            r'''var\s+stream_key\s*=\s*['"](.+?)['"]''',
+            webpage, 'stream key')
+
+        formats = [{
+            'url': 'http://spankbang.com/_%s/%s/title/%sp__mp4' % (video_id, stream_key, height),
+            'ext': 'mp4',
+            'format_id': '%sp' % height,
+            'height': int(height),
+        } for height in re.findall(r'<span[^>]+q_(\d+)p', webpage)]
+        self._sort_formats(formats)
+
+        title = self._html_search_regex(
+            r'(?s)<h1>(.+?)</h1>', webpage, 'title')
+        description = self._search_regex(
+            r'class="desc"[^>]*>([^<]+)',
+            webpage, 'description', default=None)
+        thumbnail = self._og_search_thumbnail(webpage)
+        uploader = self._search_regex(
+            r'class="user"[^>]*>([^<]+)',
+            webpage, 'uploader', fatal=False)
+
+        age_limit = self._rta_search(webpage)
 
         return {
-            "id": video_id,
-            "title": title,
-            "description": self._og_search_description(webpage),
-            "formats": formats
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'formats': formats,
+            'age_limit': age_limit,
         }
-
-# vim: tabstop=4 expandtab

From 8e4b83b96b8c2bc45aa1a9daa87c04853d3e7ade Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 6 Apr 2015 22:18:08 +0200
Subject: [PATCH 0191/1515] Remove check for ssl certs

When it uses a capath instead of a cafile, 'get_ca_certs' or 'cert_store_stats' only returns certificates already used in a connection.
(see #5364)
---
 youtube_dl/YoutubeDL.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ce4b72fd3..640b8c99d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1768,14 +1768,6 @@ class YoutubeDL(object):
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
-        # The ssl context is only available in python 2.7.9 and 3.x
-        if hasattr(https_handler, '_context'):
-            ctx = https_handler._context
-            # get_ca_certs is unavailable prior to python 3.4
-            if hasattr(ctx, 'get_ca_certs') and len(ctx.get_ca_certs()) == 0:
-                self.report_warning(
-                    'No ssl certificates were loaded, urls that use https '
-                    'won\'t work')
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             proxy_handler, https_handler, cookie_processor, ydlh)

From 29713e4268623ce5d471d5d25beea9ef433a2216 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 7 Apr 2015 14:59:13 +0200
Subject: [PATCH 0192/1515] [cnn] Match more affilliates

---
 youtube_dl/extractor/cnn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 0a77e951c..5efc5f4fe 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln|ktvk)(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z]{3,5})(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
@@ -48,6 +48,9 @@ class CNNIE(InfoExtractor):
     }, {
         'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
         'only_matching': True,
+    }, {
+        'url': 'http://cnn.com/video/?/video/us/2015/04/06/dnt-baker-refuses-anti-gay-order.wkmg',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From beb10f843f245efab24bec8ac6fead2834c5cce3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Apr 2015 21:00:22 +0600
Subject: [PATCH 0193/1515] [addanime] Add format quality (Closes #5371)

---
 youtube_dl/extractor/addanime.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 203936e54..d2a60a056 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    qualities,
 )
 
 
@@ -63,8 +64,10 @@ class AddAnimeIE(InfoExtractor):
                 note='Confirming after redirect')
             webpage = self._download_webpage(url, video_id)
 
+        FORMATS = ('normal', 'hq')
+        quality = qualities(FORMATS)
         formats = []
-        for format_id in ('normal', 'hq'):
+        for format_id in FORMATS:
             rex = r"var %s_video_file = '(.*?)';" % re.escape(format_id)
             video_url = self._search_regex(rex, webpage, 'video file URLx',
                                            fatal=False)
@@ -73,6 +76,7 @@ class AddAnimeIE(InfoExtractor):
             formats.append({
                 'format_id': format_id,
                 'url': video_url,
+                'quality': quality(format_id),
             })
         self._sort_formats(formats)
         video_title = self._og_search_title(webpage)

From 5f4b5cf044cff5263fa062596ace6180d42a4534 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Apr 2015 21:00:52 +0600
Subject: [PATCH 0194/1515] [addanime] Extend _VALID_URL (Closes #5372)

---
 youtube_dl/extractor/addanime.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index d2a60a056..28180bc3f 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class AddAnimeIE(InfoExtractor):
-    _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video\.php\?(?:.*?)v=(?P<id>[\w_]+)(?:.*)'
+    _VALID_URL = r'http://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
     _TEST = {
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',

From a35099bd331e42d96e647865f4a644b217bba62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Apr 2015 21:01:35 +0600
Subject: [PATCH 0195/1515] [addanime] Add test for #5372

---
 youtube_dl/extractor/addanime.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 28180bc3f..e3e6d2113 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class AddAnimeIE(InfoExtractor):
     _VALID_URL = r'http://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',
         'info_dict': {
@@ -26,7 +26,10 @@ class AddAnimeIE(InfoExtractor):
             'description': 'One Piece 606',
             'title': 'One Piece 606',
         }
-    }
+    }, {
+        'url': 'http://add-anime.net/video/MDUGWYKNGBD8/One-Piece-687',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From de5c54564874fd870fdfe3fd24f47e3e5f6cedf7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 03:45:02 +0800
Subject: [PATCH 0196/1515] [youtube] Skip WebVTT in DASH manifest (#5297)

---
 youtube_dl/extractor/youtube.py | 62 +++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 79ce39aa4..2774ec30b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -788,33 +788,41 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             errnote='Could not download DASH manifest')
 
         formats = []
-        for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-            url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-            if url_el is None:
-                continue
-            format_id = r.attrib['id']
-            video_url = url_el.text
-            filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
-            f = {
-                'format_id': format_id,
-                'url': video_url,
-                'width': int_or_none(r.attrib.get('width')),
-                'height': int_or_none(r.attrib.get('height')),
-                'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                'filesize': filesize,
-                'fps': int_or_none(r.attrib.get('frameRate')),
-            }
-            try:
-                existing_format = next(
-                    fo for fo in formats
-                    if fo['format_id'] == format_id)
-            except StopIteration:
-                full_info = self._formats.get(format_id, {}).copy()
-                full_info.update(f)
-                formats.append(full_info)
-            else:
-                existing_format.update(f)
+        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
+            mime_type = a.attrib.get('mimeType')
+            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
+                if url_el is None:
+                    continue
+                if mime_type == 'text/vtt':
+                    # TODO implement WebVTT downloading
+                    pass
+                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    format_id = r.attrib['id']
+                    video_url = url_el.text
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+                    f = {
+                        'format_id': format_id,
+                        'url': video_url,
+                        'width': int_or_none(r.attrib.get('width')),
+                        'height': int_or_none(r.attrib.get('height')),
+                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
+                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
+                        'filesize': filesize,
+                        'fps': int_or_none(r.attrib.get('frameRate')),
+                    }
+                    try:
+                        existing_format = next(
+                            fo for fo in formats
+                            if fo['format_id'] == format_id)
+                    except StopIteration:
+                        full_info = self._formats.get(format_id, {}).copy()
+                        full_info.update(f)
+                        formats.append(full_info)
+                    else:
+                        existing_format.update(f)
+                else:
+                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
     def _real_extract(self, url):

From cc55d08832f5008efb56228900a0dbacb428ed7f Mon Sep 17 00:00:00 2001
From: Pete Hemery <petehemery@hotmail.com>
Date: Tue, 7 Apr 2015 22:33:18 +0100
Subject: [PATCH 0197/1515] [ffmpeg] adding exception catching for call to
 os.utime in run_ffmpeg_multiple_files

---
 youtube_dl/postprocessor/ffmpeg.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 0b60ac7e7..5ef5e0e54 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -146,7 +146,11 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
-        os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
+        try:
+            os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
+        except Exception:
+            self._downloader.report_warning('Cannot update utime of file')
+
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)

From 418c5cc3fc3ea99b791ad1774a6b03504eab7086 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 17:26:51 +0800
Subject: [PATCH 0198/1515] [udn] Add new extractor

---
 test/test_utils.py               |  5 +++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/generic.py  | 20 ++++++++++
 youtube_dl/extractor/udn.py      | 66 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              | 11 ++++++
 5 files changed, 103 insertions(+)
 create mode 100644 youtube_dl/extractor/udn.py

diff --git a/test/test_utils.py b/test/test_utils.py
index 2e3a6480c..8291edd9a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     uppercase_escape,
     url_basename,
     urlencode_postdata,
+    url_infer_protocol,
     version_tuple,
     xpath_with_ns,
     xpath_text,
@@ -296,6 +297,10 @@ class TestUtil(unittest.TestCase):
             url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
             'trailer.mp4')
 
+    def test_url_infer_protocol(self):
+        self.assertEqual(url_infer_protocol('http://foo.com/', '//bar.com/'), 'http://bar.com/')
+        self.assertEqual(url_infer_protocol('http://foo.com/', 'https://bar.com/'), 'https://bar.com/')
+
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
         self.assertEqual(parse_duration(False), None)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e6fdf1297..8df1db83e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -557,6 +557,7 @@ from .udemy import (
     UdemyIE,
     UdemyCourseIE
 )
+from .udn import UDNEmbedIE
 from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2ff002643..9178d2b7b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -26,6 +26,7 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_basename,
+    url_infer_protocol,
     xpath_text,
 )
 from .brightcove import BrightcoveIE
@@ -34,6 +35,7 @@ from .ooyala import OoyalaIE
 from .rutv import RUTVIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
+from .udn import UDNEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -650,6 +652,17 @@ class GenericIE(InfoExtractor):
                 'title': "PFT Live: New leader in the 'new-look' defense",
                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
             },
+        },
+        # UDN embed
+        {
+            'url': 'http://www.udn.com/news/story/7314/822787',
+            'md5': 'de06b4c90b042c128395a88f0384817e',
+            'info_dict': {
+                'id': '300040',
+                'ext': 'mp4',
+                'title': '生物老師男變女 全校挺"做自己"',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            }
         }
     ]
 
@@ -1268,6 +1281,13 @@ class GenericIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Look for UDN embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
+        if mobj is not None:
+            return self.url_result(
+                url_infer_protocol(url, mobj.group('url')), 'UDNEmbed')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
new file mode 100644
index 000000000..2b9a733e8
--- /dev/null
+++ b/youtube_dl/extractor/udn.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+from .common import InfoExtractor
+from ..utils import (
+    url_infer_protocol,
+    js_to_json
+)
+
+
+class UDNEmbedIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//video\.udn\.com/embed/news/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://video.udn.com/embed/news/300040',
+        'md5': 'de06b4c90b042c128395a88f0384817e',
+        'info_dict': {
+            'id': '300040',
+            'ext': 'mp4',
+            'title': '生物老師男變女 全校挺"做自己"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': '//video.udn.com/embed/news/300040',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+
+        options = json.loads(js_to_json(self._html_search_regex(
+            r'var options\s*=\s*([^;]+);', page, 'video urls dictionary')))
+
+        video_urls = options['video']
+
+        if video_urls.get('youtube'):
+            return self.url_result(video_urls.get('youtube'), 'Youtube')
+
+        try:
+            del video_urls['youtube']
+        except KeyError:
+            pass
+
+        formats = [{
+            'url': self._download_webpage(
+                url_infer_protocol(url, api_url), video_id,
+                'retrieve url for %s video' % video_type),
+            'format_id': video_type,
+            'preference': 0 if video_type == 'mp4' else -1,
+        } for video_type, api_url in video_urls.items()]
+
+        self._sort_formats(formats)
+
+        thumbnail = None
+
+        if options.get('gallery') and len(options['gallery']):
+            thumbnail = options['gallery'][0].get('original')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': options['title'],
+            'thumbnail': thumbnail
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52f0dd09a..f3b8d9f81 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1711,6 +1711,17 @@ def determine_protocol(info_dict):
     return compat_urllib_parse_urlparse(url).scheme
 
 
+def url_infer_protocol(ref_url, target_url):
+    """ Infer protocol for protocol independent target urls """
+    parsed_target_url = list(compat_urllib_parse_urlparse(target_url))
+    if parsed_target_url[0]:
+        return target_url
+
+    parsed_target_url[0] = compat_urllib_parse_urlparse(ref_url).scheme
+
+    return compat_urlparse.urlunparse(parsed_target_url)
+
+
 def render_table(header_row, data):
     """ Render a list of rows, each as a list of values """
     table = [header_row] + data

From 4a20c9f628b6038766a76ed1c3e37aa6a2b34718 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 17:42:26 +0800
Subject: [PATCH 0199/1515] [livestream] Extend _VALID_URL (fixes #5375)

---
 youtube_dl/extractor/livestream.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 2467f8bdd..ec309dadd 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -21,7 +21,7 @@ from ..utils import (
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'https?://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:new\.)?livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
@@ -51,6 +51,9 @@ class LivestreamIE(InfoExtractor):
     }, {
         'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
         'only_matching': True,
+    }, {
+        'url': 'http://livestream.com/bsww/concacafbeachsoccercampeonato2015',
+        'only_matching': True,
     }]
 
     def _parse_smil(self, video_id, smil_url):

From bd7a6478a2db228c7325a48e13e0e699502f56f4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 19:20:34 +0800
Subject: [PATCH 0200/1515] [theplatform] Fix video url extraction (fixes
 #5340)

In SMIL 2.1, <switch> nodes may be enclosed in <par>. See
http://www.w3.org/TR/SMIL2/smil-timing.html#edef-par
---
 youtube_dl/extractor/theplatform.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0e3e627f4..a7d060459 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -28,7 +28,7 @@ class ThePlatformIE(InfoExtractor):
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
-    _TEST = {
+    _TESTS = [{
         # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/
         'url': 'http://link.theplatform.com/s/dJ5BDC/e9I_cZgTgIPd/meta.smil?format=smil&Tracking=true&mbr=true',
         'info_dict': {
@@ -42,7 +42,20 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-    }
+        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
+    }, {
+        'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
+        'info_dict': {
+            'id': '22d_qsQ6MIRT',
+            'ext': 'flv',
+            'description': 'md5:ac330c9258c04f9d7512cf26b9595409',
+            'title': 'Tesla Model S: A second step towards a cleaner motoring future',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }]
 
     @staticmethod
     def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
@@ -126,7 +139,7 @@ class ThePlatformIE(InfoExtractor):
             formats = self._extract_f4m_formats(f4m_url, video_id)
         else:
             formats = []
-            switch = body.find(_x('smil:switch'))
+            switch = body.find(_x('.//smil:switch'))
             if switch is not None:
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):

From a662163fd5b3c9b1221b8aeaf54ed9083af8574f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 20:21:34 +0800
Subject: [PATCH 0201/1515] [theplatform] Rework on <switch> inside <par>

---
 youtube_dl/extractor/theplatform.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index a7d060459..bcddd23e4 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -139,7 +139,9 @@ class ThePlatformIE(InfoExtractor):
             formats = self._extract_f4m_formats(f4m_url, video_id)
         else:
             formats = []
-            switch = body.find(_x('.//smil:switch'))
+            switch = body.find(_x('smil:switch'))
+            if switch is None:
+                switch = body.find(_x('smil:par//smil:switch'))
             if switch is not None:
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):

From 0a1603634bcf799eeb769d95b6e716e66123b77f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 21:39:34 +0800
Subject: [PATCH 0202/1515] [utils] Remove url_infer_protocol

---
 test/test_utils.py              |  5 -----
 youtube_dl/extractor/generic.py |  3 +--
 youtube_dl/extractor/udn.py     |  8 +++-----
 youtube_dl/utils.py             | 11 -----------
 4 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8291edd9a..2e3a6480c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,7 +53,6 @@ from youtube_dl.utils import (
     uppercase_escape,
     url_basename,
     urlencode_postdata,
-    url_infer_protocol,
     version_tuple,
     xpath_with_ns,
     xpath_text,
@@ -297,10 +296,6 @@ class TestUtil(unittest.TestCase):
             url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
             'trailer.mp4')
 
-    def test_url_infer_protocol(self):
-        self.assertEqual(url_infer_protocol('http://foo.com/', '//bar.com/'), 'http://bar.com/')
-        self.assertEqual(url_infer_protocol('http://foo.com/', 'https://bar.com/'), 'https://bar.com/')
-
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
         self.assertEqual(parse_duration(False), None)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9178d2b7b..6c212efac 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -26,7 +26,6 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_basename,
-    url_infer_protocol,
     xpath_text,
 )
 from .brightcove import BrightcoveIE
@@ -1286,7 +1285,7 @@ class GenericIE(InfoExtractor):
             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
-                url_infer_protocol(url, mobj.group('url')), 'UDNEmbed')
+                compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index 2b9a733e8..bba25bb58 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -3,10 +3,8 @@ from __future__ import unicode_literals
 
 import json
 from .common import InfoExtractor
-from ..utils import (
-    url_infer_protocol,
-    js_to_json
-)
+from ..utils import js_to_json
+from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
@@ -45,7 +43,7 @@ class UDNEmbedIE(InfoExtractor):
 
         formats = [{
             'url': self._download_webpage(
-                url_infer_protocol(url, api_url), video_id,
+                compat_urlparse.urljoin(url, api_url), video_id,
                 'retrieve url for %s video' % video_type),
             'format_id': video_type,
             'preference': 0 if video_type == 'mp4' else -1,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f3b8d9f81..52f0dd09a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1711,17 +1711,6 @@ def determine_protocol(info_dict):
     return compat_urllib_parse_urlparse(url).scheme
 
 
-def url_infer_protocol(ref_url, target_url):
-    """ Infer protocol for protocol independent target urls """
-    parsed_target_url = list(compat_urllib_parse_urlparse(target_url))
-    if parsed_target_url[0]:
-        return target_url
-
-    parsed_target_url[0] = compat_urllib_parse_urlparse(ref_url).scheme
-
-    return compat_urlparse.urlunparse(parsed_target_url)
-
-
 def render_table(header_row, data):
     """ Render a list of rows, each as a list of values """
     table = [header_row] + data

From aef8fdba1172d60983ba9685249c03b66e7a94f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 21:03:11 +0600
Subject: [PATCH 0203/1515] [theplatform] Allow <par> without <swtich> at all

Bare `wget` on http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRTl results in an XML without <switch> at all
but with <par> and <video> inside it. Let's handle this possible outcome as well.
---
 youtube_dl/extractor/theplatform.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index bcddd23e4..2d2178331 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -142,6 +142,8 @@ class ThePlatformIE(InfoExtractor):
             switch = body.find(_x('smil:switch'))
             if switch is None:
                 switch = body.find(_x('smil:par//smil:switch'))
+            if switch is None:
+                switch = body.find(_x('smil:par'))
             if switch is not None:
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):

From dd29eb7f816bae7cf7807db9f26b7f5621b8d557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 21:40:31 +0600
Subject: [PATCH 0204/1515] [postprocessor/common:postprocessor/ffmpeg]
 Generalize utime

---
 youtube_dl/postprocessor/common.py | 13 ++++++++++++-
 youtube_dl/postprocessor/ffmpeg.py | 12 ++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index e54ae678d..ef9fdfa19 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,6 +1,11 @@
 from __future__ import unicode_literals
 
-from ..utils import PostProcessingError
+import os
+
+from ..utils import (
+    PostProcessingError,
+    encodeFilename,
+)
 
 
 class PostProcessor(object):
@@ -46,6 +51,12 @@ class PostProcessor(object):
         """
         return None, information  # by default, keep file and do nothing
 
+    def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
+        try:
+            os.utime(encodeFilename(path), (atime, mtime))
+        except Exception:
+            self._downloader.report_warning(errnote)
+
 
 class AudioConversionError(PostProcessingError):
     pass
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5ef5e0e54..8e99a3c2c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -146,10 +146,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
-        try:
-            os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
-        except Exception:
-            self._downloader.report_warning('Cannot update utime of file')
+        self.try_utime(out_path, oldest_mtime, oldest_mtime)
 
         if self._deletetempfiles:
             for ipath in input_paths:
@@ -284,10 +281,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
         # Try to update the date time for extracted audio file.
         if information.get('filetime') is not None:
-            try:
-                os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-            except Exception:
-                self._downloader.report_warning('Cannot update utime of audio file')
+            self.try_utime(
+                new_path, time.time(), information['filetime'],
+                errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
         return self._nopostoverwrites, information

From 372f08c99057b6a994c72cf2591e184231b3e850 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 22:27:25 +0600
Subject: [PATCH 0205/1515] [theplatform] Fix for python 2.6

At least single depth level extraction...
---
 youtube_dl/extractor/theplatform.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 2d2178331..50c51d825 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -42,8 +42,8 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
     }, {
+        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
         'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
         'info_dict': {
             'id': '22d_qsQ6MIRT',
@@ -128,7 +128,7 @@ class ThePlatformIE(InfoExtractor):
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))
 
-        f4m_node = body.find(_x('smil:seq//smil:video'))
+        f4m_node = body.find(_x('smil:seq//smil:video')) or body.find(_x('smil:seq/smil:video'))
         if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
             f4m_url = f4m_node.attrib['src']
             if 'manifest.f4m?' not in f4m_url:
@@ -141,7 +141,7 @@ class ThePlatformIE(InfoExtractor):
             formats = []
             switch = body.find(_x('smil:switch'))
             if switch is None:
-                switch = body.find(_x('smil:par//smil:switch'))
+                switch = body.find(_x('smil:par//smil:switch')) or body.find(_x('smil:par/smil:switch'))
             if switch is None:
                 switch = body.find(_x('smil:par'))
             if switch is not None:
@@ -162,7 +162,7 @@ class ThePlatformIE(InfoExtractor):
                         'vbr': vbr,
                     })
             else:
-                switch = body.find(_x('smil:seq//smil:switch'))
+                switch = body.find(_x('smil:seq//smil:switch')) or body.find(_x('smil:seq/smil:switch'))
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
                     vbr = int(attr['system-bitrate']) // 1000

From 402a3efc927538684fb69e23c65a97b19ce4f663 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 22:29:10 +0600
Subject: [PATCH 0206/1515] [theplatform] Modernize

---
 youtube_dl/extractor/theplatform.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 50c51d825..6a006b2d2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -17,6 +17,7 @@ from ..utils import (
     ExtractorError,
     xpath_with_ns,
     unsmuggle_url,
+    int_or_none,
 )
 
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
@@ -148,9 +149,9 @@ class ThePlatformIE(InfoExtractor):
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
-                    width = int(attr['width'])
-                    height = int(attr['height'])
-                    vbr = int(attr['system-bitrate']) // 1000
+                    width = int_or_none(attr.get('width'))
+                    height = int_or_none(attr.get('height'))
+                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
                     format_id = '%dx%d_%dk' % (width, height, vbr)
                     formats.append({
                         'format_id': format_id,
@@ -165,7 +166,7 @@ class ThePlatformIE(InfoExtractor):
                 switch = body.find(_x('smil:seq//smil:switch')) or body.find(_x('smil:seq/smil:switch'))
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
-                    vbr = int(attr['system-bitrate']) // 1000
+                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
                     ext = determine_ext(attr['src'])
                     if ext == 'once':
                         ext = 'mp4'
@@ -184,5 +185,5 @@ class ThePlatformIE(InfoExtractor):
             'formats': formats,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
-            'duration': info['duration'] // 1000,
+            'duration': int_or_none(info.get('duration'), 1000),
         }

From af76e8174d1f73e0c44e6bead56a780c665e299d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 02:25:31 +0600
Subject: [PATCH 0207/1515] [dailymotion:user] Improve _VALID_URL (Closes
 #5380)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 47d58330b..7615ecd4b 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -224,7 +224,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:old/)?user/(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1dc2726f8da5ec7726e3c45ee718ce4464b83ead Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 9 Apr 2015 00:21:19 +0200
Subject: [PATCH 0208/1515] release 2015.04.09

---
 README.md              | 194 ++++++++++++++++++++---------------------
 docs/supportedsites.md |   5 ++
 youtube_dl/version.py  |   2 +-
 3 files changed, 103 insertions(+), 98 deletions(-)

diff --git a/README.md b/README.md
index 4f9fc8174..caa1478d9 100644
--- a/README.md
+++ b/README.md
@@ -45,21 +45,21 @@ which means you can modify it, redistribute it or use it however you like.
     youtube-dl [OPTIONS] URL [URL...]
 
 # OPTIONS
-    -h, --help                       print this help text and exit
-    --version                        print program version and exit
-    -U, --update                     update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
-    -i, --ignore-errors              continue on download errors, for example to skip unavailable videos in a playlist
+    -h, --help                       Print this help text and exit
+    --version                        Print program version and exit
+    -U, --update                     Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
+    -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
     --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
-    --dump-user-agent                display the current browser identification
+    --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors and the URLs they would handle
     --extractor-descriptions         Output descriptions of all supported extractors
-    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple".
+    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
                                      default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
     --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
                                      in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
     --flat-playlist                  Do not extract the videos of a playlist, only list them.
-    --no-color                       Do not emit color codes in output.
+    --no-color                       Do not emit color codes in output
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
@@ -71,70 +71,70 @@ which means you can modify it, redistribute it or use it however you like.
                                      not present) is used for the actual downloading. (experimental)
 
 ## Video Selection:
-    --playlist-start NUMBER          playlist video to start at (default is 1)
-    --playlist-end NUMBER            playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+    --playlist-start NUMBER          Playlist video to start at (default is 1)
+    --playlist-end NUMBER            Playlist video to end at (default is last)
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
                                      if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
                                      download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              download only matching titles (regex or caseless sub-string)
-    --reject-title REGEX             skip download for matching titles (regex or caseless sub-string)
+    --match-title REGEX              Download only matching titles (regex or caseless sub-string)
+    --reject-title REGEX             Skip download for matching titles (regex or caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
     --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
     --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
-    --date DATE                      download only videos uploaded in this date
-    --datebefore DATE                download only videos uploaded on or before this date (i.e. inclusive)
-    --dateafter DATE                 download only videos uploaded on or after this date (i.e. inclusive)
+    --date DATE                      Download only videos uploaded in this date
+    --datebefore DATE                Download only videos uploaded on or before this date (i.e. inclusive)
+    --dateafter DATE                 Download only videos uploaded on or after this date (i.e. inclusive)
     --min-views COUNT                Do not download any videos with less than COUNT views
     --max-views COUNT                Do not download any videos with more than COUNT views
-    --match-filter FILTER            (Experimental) Generic video filter. Specify any key (see help for -o for a list of available keys) to match if the key is present,
+    --match-filter FILTER            Generic video filter (experimental). Specify any key (see help for -o for a list of available keys) to match if the key is present,
                                      !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
                                      a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
                                      operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
                                      functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    If the URL refers to a video and a playlist, download only the video.
-    --yes-playlist                   If the URL refers to a video and a playlist, download the playlist.
-    --age-limit YEARS                download only videos suitable for the given age
+    --no-playlist                    Download only the video, if the URL refers to a video and a playlist.
+    --yes-playlist                   Download the playlist, if the URL refers to a video and a playlist.
+    --age-limit YEARS                Download only videos suitable for the given age
     --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
     --include-ads                    Download advertisements as well (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           maximum download rate in bytes per second (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10), or "infinite".
-    --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K) (default is 1024)
-    --no-resize-buffer               do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
+    -r, --rate-limit LIMIT           Maximum download rate in bytes per second (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            Number of retries (default is 10), or "infinite".
+    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K) (default is 1024)
+    --no-resize-buffer               Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             (experimental) set file xattribute ytdl.filesize with expected filesize
-    --hls-prefer-native              (experimental) Use the native HLS downloader instead of ffmpeg.
+    --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
+    --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
     --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
-    --external-downloader-args ARGS  Give these arguments to the external downloader.
+    --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
-    -a, --batch-file FILE            file containing URLs to download ('-' for stdin)
-    --id                             use only video ID in file name
-    -o, --output TEMPLATE            output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
+    -a, --batch-file FILE            File containing URLs to download ('-' for stdin)
+    --id                             Use only video ID in file name
+    -o, --output TEMPLATE            Output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
                                      nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
-                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube's itags: "137"),
+                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like YouTube's itags: "137"),
                                      %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
                                      %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
                                      %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
                                      %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
                                      Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
+    --autonumber-size NUMBER         Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
     --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
-    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000
-    -t, --title                      [deprecated] use title in file name (default)
-    -l, --literal                    [deprecated] alias of --title
-    -w, --no-overwrites              do not overwrite files
-    -c, --continue                   force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
-    --no-continue                    do not resume partially downloaded files (restart from beginning)
-    --no-part                        do not use .part files - write directly into output file
-    --no-mtime                       do not use the Last-modified header to set the file modification time
-    --write-description              write video description to a .description file
-    --write-info-json                write video metadata to a .info.json file
-    --write-annotations              write video annotations to a .annotation file
-    --load-info FILE                 json file containing the video information (created with the "--write-json" option)
-    --cookies FILE                   file to read cookies from and dump cookie jar in
+    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000
+    -t, --title                      [deprecated] Use title in file name (default)
+    -l, --literal                    [deprecated] Alias of --title
+    -w, --no-overwrites              Do not overwrite files
+    -c, --continue                   Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
+    --no-continue                    Do not resume partially downloaded files (restart from beginning)
+    --no-part                        Do not use .part files - write directly into output file
+    --no-mtime                       Do not use the Last-modified header to set the file modification time
+    --write-description              Write video description to a .description file
+    --write-info-json                Write video metadata to a .info.json file
+    --write-annotations              Write video annotations to a .annotation file
+    --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
+    --cookies FILE                   File to read cookies from and dump cookie jar in
     --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
                                      or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
                                      change.
@@ -142,49 +142,49 @@ which means you can modify it, redistribute it or use it however you like.
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
-    --write-thumbnail                write thumbnail image to disk
-    --write-all-thumbnails           write all thumbnail image formats to disk
+    --write-thumbnail                Write thumbnail image to disk
+    --write-all-thumbnails           Write all thumbnail image formats to disk
     --list-thumbnails                Simulate and list all available thumbnail formats
 
 ## Verbosity / Simulation Options:
-    -q, --quiet                      activates quiet mode
+    -q, --quiet                      Activate quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   do not download the video and do not write anything to disk
-    --skip-download                  do not download the video
-    -g, --get-url                    simulate, quiet but print URL
-    -e, --get-title                  simulate, quiet but print title
-    --get-id                         simulate, quiet but print id
-    --get-thumbnail                  simulate, quiet but print thumbnail URL
-    --get-description                simulate, quiet but print video description
-    --get-duration                   simulate, quiet but print video length
-    --get-filename                   simulate, quiet but print output filename
-    --get-format                     simulate, quiet but print output format
-    -j, --dump-json                  simulate, quiet but print JSON information. See --output for a description of available keys.
-    -J, --dump-single-json           simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
+    -s, --simulate                   Do not download the video and do not write anything to disk
+    --skip-download                  Do not download the video
+    -g, --get-url                    Simulate, quiet but print URL
+    -e, --get-title                  Simulate, quiet but print title
+    --get-id                         Simulate, quiet but print id
+    --get-thumbnail                  Simulate, quiet but print thumbnail URL
+    --get-description                Simulate, quiet but print video description
+    --get-duration                   Simulate, quiet but print video length
+    --get-filename                   Simulate, quiet but print output filename
+    --get-format                     Simulate, quiet but print output format
+    -j, --dump-json                  Simulate, quiet but print JSON information. See --output for a description of available keys.
+    -J, --dump-single-json           Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
                                      information in a single line.
     --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
-    --newline                        output progress bar as new lines
-    --no-progress                    do not print progress bar
-    --console-title                  display progress in console titlebar
-    -v, --verbose                    print various debugging information
-    --dump-pages                     print downloaded pages to debug problems (very verbose)
+    --newline                        Output progress bar as new lines
+    --no-progress                    Do not print progress bar
+    --console-title                  Display progress in console titlebar
+    -v, --verbose                    Print various debugging information
+    --dump-pages                     Print downloaded pages to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
-    -C, --call-home                  Contact the youtube-dl server for debugging.
-    --no-call-home                   Do NOT contact the youtube-dl server for debugging.
+    -C, --call-home                  Contact the youtube-dl server for debugging
+    --no-call-home                   Do NOT contact the youtube-dl server for debugging
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
-    --no-check-certificate           Suppress HTTPS certificate validation.
+    --no-check-certificate           Suppress HTTPS certificate validation
     --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
-    --user-agent UA                  specify a custom user agent
-    --referer URL                    specify a custom referer, use if the video access is restricted to one domain
-    --add-header FIELD:VALUE         specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
+    --user-agent UA                  Specify a custom user agent
+    --referer URL                    Specify a custom referer, use if the video access is restricted to one domain
+    --add-header FIELD:VALUE         Specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
     --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
     --sleep-interval SECONDS         Number of seconds to sleep before each download.
 
 ## Video Format Options:
-    -f, --format FORMAT              video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
+    -f, --format FORMAT              Video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
                                      extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio",
                                      "worst".  You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]").
                                      This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec,
@@ -194,44 +194,44 @@ which means you can modify it, redistribute it or use it however you like.
                                      Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio
                                      of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f
                                      bestvideo+bestaudio.
-    --all-formats                    download all available video formats
-    --prefer-free-formats            prefer free video formats unless a specific one is requested
-    --max-quality FORMAT             highest quality format to download
-    -F, --list-formats               list all available formats
+    --all-formats                    Download all available video formats
+    --prefer-free-formats            Prefer free video formats unless a specific one is requested
+    --max-quality FORMAT             Highest quality format to download
+    -F, --list-formats               List all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
     --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
                                      merge is required
 
 ## Subtitle Options:
-    --write-sub                      write subtitle file
-    --write-auto-sub                 write automatic subtitle file (youtube only)
-    --all-subs                       downloads all the available subtitles of the video
-    --list-subs                      lists all available subtitles for the video
-    --sub-format FORMAT              subtitle format, accepts formats preference, for example: "ass/srt/best"
-    --sub-lang LANGS                 languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
+    --write-sub                      Write subtitle file
+    --write-auto-sub                 Write automatic subtitle file (YouTube only)
+    --all-subs                       Download all the available subtitles of the video
+    --list-subs                      List all available subtitles for the video
+    --sub-format FORMAT              Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"
+    --sub-lang LANGS                 Languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
 
 ## Authentication Options:
-    -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password. If this option is left out, youtube-dl will ask interactively.
-    -2, --twofactor TWOFACTOR        two-factor auth code
-    -n, --netrc                      use .netrc authentication data
-    --video-password PASSWORD        video password (vimeo, smotri)
+    -u, --username USERNAME          Login with this account ID
+    -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
+    -2, --twofactor TWOFACTOR        Two-factor auth code
+    -n, --netrc                      Use .netrc authentication data
+    --video-password PASSWORD        Video password (vimeo, smotri)
 
 ## Post-processing Options:
-    -x, --extract-audio              convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K
-                                     (default 5)
+    -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
+    --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
+    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
+                                     5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
-    -k, --keep-video                 keeps the video file on disk after the post-processing; the video is erased by default
-    --no-post-overwrites             do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     embed subtitles in the video (only for mp4 videos)
-    --embed-thumbnail                embed thumbnail in the audio as cover art
-    --add-metadata                   write metadata to the video file
-    --metadata-from-title FORMAT     parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
+    -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
+    --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
+    --embed-subs                     Embed subtitles in the video (only for mp4 videos)
+    --embed-thumbnail                Embed thumbnail in the audio as cover art
+    --add-metadata                   Write metadata to the video file
+    --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
                                      parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
-    --xattrs                         write metadata to the video file's xattrs (using dublin core and xdg standards)
+    --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
                                      fix file if we can, warn otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2785b9587..c85a39918 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -166,6 +166,7 @@
  - **Gamekings**
  - **GameOne**
  - **gameone:playlist**
+ - **Gamersyde**
  - **GameSpot**
  - **GameStar**
  - **Gametrailers**
@@ -351,6 +352,7 @@
  - **PornHub**
  - **PornHubPlaylist**
  - **Pornotube**
+ - **PornoVoisines**
  - **PornoXO**
  - **PrimeShareTV**
  - **PromptFile**
@@ -362,6 +364,7 @@
  - **radio.de**
  - **radiobremen**
  - **radiofrance**
+ - **RadioJavan**
  - **Rai**
  - **RBMARadio**
  - **RedTube**
@@ -422,6 +425,7 @@
  - **southpark.cc.com**
  - **southpark.de**
  - **Space**
+ - **SpankBang**
  - **Spankwire**
  - **Spiegel**
  - **Spiegel:Article**: Articles on spiegel.de
@@ -501,6 +505,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
+ - **UDNEmbed**
  - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e1c385bec..1095fea2f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.03'
+__version__ = '2015.04.09'

From ce73839fe4bc1ac43d7a6540df040139f82948b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 9 Apr 2015 14:01:33 +0200
Subject: [PATCH 0209/1515] [rtve] Detect videos that are no longer available

---
 youtube_dl/extractor/rtve.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 8d9be1b98..849300140 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -8,6 +8,7 @@ import time
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    ExtractorError,
     float_or_none,
     remove_end,
     std_headers,
@@ -98,6 +99,8 @@ class RTVEALaCartaIE(InfoExtractor):
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
+        if info['state'] == 'DESPU':
+            raise ExtractorError('The video is no longer available', expected=True)
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)

From aa2af7ba7469370f987aa178ac031fdad00aae3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 19:53:00 +0600
Subject: [PATCH 0210/1515] [dumpert] Add nsfw cookie (Closes #5382)

---
 youtube_dl/extractor/dumpert.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index e43bc81b2..9c594b757 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import qualities
 
 
@@ -23,7 +24,10 @@ class DumpertIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Cookie', 'nsfw=1')
+        webpage = self._download_webpage(req, video_id)
 
         files_base64 = self._search_regex(
             r'data-files="([^"]+)"', webpage, 'data files')

From 6ac41a4ef50029fa3b0eee5083805d0133a4d1dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:32:22 +0600
Subject: [PATCH 0211/1515] [vine] Zero rate videos is perfectly valid (#5389)

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c3187cfeb..353791e1d 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -37,7 +37,7 @@ class VineIE(InfoExtractor):
             'vcodec': f['format'],
             'quality': f['rate'],
             'url': f['videoUrl'],
-        } for f in data['videoUrls'] if f.get('rate')]
+        } for f in data['videoUrls']]
 
         self._sort_formats(formats)
 

From 58a9f1b86404e3a6c26c7143cb408db793aa5946 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:32:48 +0600
Subject: [PATCH 0212/1515] [vine] Fix post data regex (Closes #5389)

---
 youtube_dl/extractor/vine.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 353791e1d..13c13152e 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -30,7 +30,8 @@ class VineIE(InfoExtractor):
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
         data = json.loads(self._html_search_regex(
-            r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
+            r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+            webpage, 'vine data'))
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,

From 3359fb661fdf3f9308611dc10c5b579b7a7ef56e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:37:54 +0600
Subject: [PATCH 0213/1515] [vine] Add tests for #5389

---
 youtube_dl/extractor/vine.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 13c13152e..804adbdb0 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -10,7 +10,7 @@ from ..utils import unified_strdate
 
 class VineIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vine\.co/v/(?P<id>\w+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vine.co/v/b9KOOWX7HUx',
         'md5': '2f36fed6235b16da96ce9b4dc890940d',
         'info_dict': {
@@ -23,7 +23,33 @@ class VineIE(InfoExtractor):
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
         },
-    }
+    }, {
+        'url': 'https://vine.co/v/MYxVapFvz2z',
+        'md5': '7b9a7cbc76734424ff942eb52c8f1065',
+        'info_dict': {
+            'id': 'MYxVapFvz2z',
+            'ext': 'mp4',
+            'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
+            'alt_title': 'Vine by Luna',
+            'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
+            'upload_date': '20140815',
+            'uploader': 'Luna',
+            'uploader_id': '1102363502380728320',
+        },
+    }, {
+        'url': 'https://vine.co/v/bxVjBbZlPUH',
+        'md5': 'ea27decea3fa670625aac92771a96b73',
+        'info_dict': {
+            'id': 'bxVjBbZlPUH',
+            'ext': 'mp4',
+            'title': '#mw3 #ac130 #killcam #angelofdeath',
+            'alt_title': 'Vine by Z3k3',
+            'description': '#mw3 #ac130 #killcam #angelofdeath',
+            'upload_date': '20130430',
+            'uploader': 'Z3k3',
+            'uploader_id': '936470460173008896',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 64f1aba8f1a6dba88e0dd0edc799fee978c7ce76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:40:18 +0600
Subject: [PATCH 0214/1515] [vine] Extend _VALID_URL

---
 youtube_dl/extractor/vine.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 804adbdb0..a4d5af147 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -9,7 +9,7 @@ from ..utils import unified_strdate
 
 
 class VineIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vine\.co/v/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?vine\.co/(?:v|oembed)/(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://vine.co/v/b9KOOWX7HUx',
         'md5': '2f36fed6235b16da96ce9b4dc890940d',
@@ -49,6 +49,9 @@ class VineIE(InfoExtractor):
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
         },
+    }, {
+        'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4c4780c25e20d6ad281faff5d6aede3df4b58f66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:41:41 +0600
Subject: [PATCH 0215/1515] [vine] Modernize

---
 youtube_dl/extractor/vine.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index a4d5af147..d4f5a991e 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -58,9 +58,11 @@ class VineIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
-        data = json.loads(self._html_search_regex(
-            r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
-            webpage, 'vine data'))
+        data = self._parse_json(
+            self._html_search_regex(
+                r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+                webpage, 'vine data'),
+            video_id)
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,

From ce9f47de99f3f607ad940897e457d02ea084795d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 9 Apr 2015 23:54:53 +0300
Subject: [PATCH 0216/1515] [teamcoco] Fix extraction

---
 youtube_dl/extractor/teamcoco.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index a46a7ecba..b3cfe4e4f 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -4,7 +4,10 @@ import base64
 import re
 
 from .common import InfoExtractor
-from ..utils import qualities
+from ..utils import (
+    ExtractorError,
+    qualities,
+)
 
 
 class TeamcocoIE(InfoExtractor):
@@ -49,14 +52,12 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        embed_url = 'http://teamcoco.com/embed/v/%s' % video_id
-        embed = self._download_webpage(
-            embed_url, video_id, 'Downloading embed page')
-
-        player_data = self._parse_json(self._search_regex(
-            r'Y\.Ginger\.Module\.Player(?:;var\s*player\s*=\s*new\s*m)?\((\{.*?\})\);', embed, 'player data'), video_id)
+        preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
+        if not preloads:
+            raise ExtractorError('Preload information could not be extracted')
+        preload = max([(len(p), p) for p in preloads])[1]
         data = self._parse_json(
-            base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id)
+            base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From 5bb6328cb944f67f7981eddc40d92998a153f00d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 9 Apr 2015 23:57:51 +0300
Subject: [PATCH 0217/1515] [teamcoco] Extract m3u8 URLs

---
 youtube_dl/extractor/teamcoco.py | 34 ++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index b3cfe4e4f..41677503c 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -62,23 +62,27 @@ class TeamcocoIE(InfoExtractor):
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
-            m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
-            if m_format is not None:
-                format_id = m_format.group(1)
+            if filed['type'] == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    filed['url'], video_id, ext='mp4'))
             else:
-                format_id = filed['bitrate']
-            tbr = (
-                int(filed['bitrate'])
-                if filed['bitrate'].isdigit()
-                else None)
+                m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
+                if m_format is not None:
+                    format_id = m_format.group(1)
+                else:
+                    format_id = filed['bitrate']
+                tbr = (
+                    int(filed['bitrate'])
+                    if filed['bitrate'].isdigit()
+                    else None)
 
-            formats.append({
-                'url': filed['url'],
-                'ext': 'mp4',
-                'tbr': tbr,
-                'format_id': format_id,
-                'quality': get_quality(format_id),
-            })
+                formats.append({
+                    'url': filed['url'],
+                    'ext': 'mp4',
+                    'tbr': tbr,
+                    'format_id': format_id,
+                    'quality': get_quality(format_id),
+                })
 
         self._sort_formats(formats)
 

From 7088f5b5fa07381ec9d484c8ef83616724654e3f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 10 Apr 2015 02:03:38 +0300
Subject: [PATCH 0218/1515] [teamcoco] Extract duration

---
 youtube_dl/extractor/teamcoco.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 41677503c..1caf08cb7 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -21,6 +21,7 @@ class TeamcocoIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
                 'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
+                'duration': 504,
                 'age_limit': 0,
             }
         }, {
@@ -31,6 +32,7 @@ class TeamcocoIE(InfoExtractor):
                 'ext': 'mp4',
                 'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
                 'title': 'Louis C.K. Interview Pt. 1 11/3/11',
+                'duration': 288,
                 'age_limit': 0,
             }
         }
@@ -93,5 +95,6 @@ class TeamcocoIE(InfoExtractor):
             'title': data['title'],
             'thumbnail': data.get('thumb', {}).get('href'),
             'description': data.get('teaser'),
+            'duration': data.get('duration'),
             'age_limit': self._family_friendly_search(webpage),
         }

From 8749477ed0a3cbc85d1726b6526fa5e794ce6072 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 22:27:16 +0600
Subject: [PATCH 0219/1515] [rai] Fix extraction (Closes #5396)

---
 youtube_dl/extractor/rai.py | 72 ++++++++++++++++++++++++++-----------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 144e33982..115cc64cc 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class RaiIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
+    _VALID_URL = r'(?P<url>(?P<host>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it))/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
@@ -64,32 +64,65 @@ class RaiIE(InfoExtractor):
         },
     ]
 
+    def _extract_relinker_url(self, webpage):
+        return self._proto_relative_url(self._search_regex(
+            [r'name="videourl" content="([^"]+)"', r'var\s+videoURL(?:_MP4)?\s*=\s*"([^"]+)"'],
+            webpage, 'relinker url', default=None))
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        host = mobj.group('host')
 
-        media = self._download_json('%s?json' % mobj.group('url'), video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
 
-        title = media.get('name')
-        description = media.get('desc')
-        thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
-        duration = parse_duration(media.get('length'))
-        uploader = media.get('author')
-        upload_date = unified_strdate(media.get('date'))
+        relinker_url = self._extract_relinker_url(webpage)
 
-        formats = []
+        if not relinker_url:
+            iframe_path = self._search_regex(
+                r'<iframe[^>]+src="/?(dl/[^"]+\?iframe\b[^"]*)"',
+                webpage, 'iframe')
+            iframe_page = self._download_webpage(
+                '%s/%s' % (host, iframe_path), video_id)
+            relinker_url = self._extract_relinker_url(iframe_page)
 
-        for format_id in ['wmv', 'm3u8', 'mediaUri', 'h264']:
-            media_url = media.get(format_id)
-            if not media_url:
-                continue
-            formats.append({
+        relinker = self._download_json(
+            '%s&output=47' % relinker_url, video_id)
+
+        media_url = relinker['video'][0]
+        ct = relinker.get('ct')
+        if ct == 'f4m':
+            formats = self._extract_f4m_formats(
+                media_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id)
+        else:
+            formats = [{
                 'url': media_url,
-                'format_id': format_id,
-                'ext': 'mp4',
-            })
+                'format_id': ct,
+            }]
 
-        subtitles = self.extract_subtitles(video_id, url)
+        json_link = self._html_search_meta(
+            'jsonlink', webpage, 'JSON link', default=None)
+        if json_link:
+            media = self._download_json(
+                host + json_link, video_id, 'Downloading video JSON')
+            title = media.get('name')
+            description = media.get('desc')
+            thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
+            duration = parse_duration(media.get('length'))
+            uploader = media.get('author')
+            upload_date = unified_strdate(media.get('date'))
+        else:
+            title = self._search_regex(
+                r'var\s+videoTitolo\s*=\s*"([^"]+)";',
+                webpage, 'title', default=None) or self._og_search_title(webpage)
+            description = self._og_search_description(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
+            duration = None
+            uploader = self._html_search_meta('Editore', webpage, 'uploader')
+            upload_date = unified_strdate(self._html_search_meta(
+                'item-date', webpage, 'upload date'))
+
+        subtitles = self.extract_subtitles(video_id, webpage)
 
         return {
             'id': video_id,
@@ -103,8 +136,7 @@ class RaiIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _get_subtitles(self, video_id, url):
-        webpage = self._download_webpage(url, video_id)
+    def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
         if m:

From d7c78decb0fb4adcf84e5e74cd0d4f858d48e2ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 22:41:26 +0600
Subject: [PATCH 0220/1515] [rai] Improve extraction

---
 youtube_dl/extractor/rai.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 115cc64cc..12127c634 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -82,9 +82,9 @@ class RaiIE(InfoExtractor):
             iframe_path = self._search_regex(
                 r'<iframe[^>]+src="/?(dl/[^"]+\?iframe\b[^"]*)"',
                 webpage, 'iframe')
-            iframe_page = self._download_webpage(
+            webpage = self._download_webpage(
                 '%s/%s' % (host, iframe_path), video_id)
-            relinker_url = self._extract_relinker_url(iframe_page)
+            relinker_url = self._extract_relinker_url(webpage)
 
         relinker = self._download_json(
             '%s&output=47' % relinker_url, video_id)
@@ -112,15 +112,15 @@ class RaiIE(InfoExtractor):
             uploader = media.get('author')
             upload_date = unified_strdate(media.get('date'))
         else:
-            title = self._search_regex(
-                r'var\s+videoTitolo\s*=\s*"([^"]+)";',
-                webpage, 'title', default=None) or self._og_search_title(webpage)
+            title = (self._search_regex(
+                r'var\s+videoTitolo\s*=\s*"(.+?)";',
+                webpage, 'title', default=None) or self._og_search_title(webpage)).replace('\\"', '"')
             description = self._og_search_description(webpage)
             thumbnail = self._og_search_thumbnail(webpage)
             duration = None
             uploader = self._html_search_meta('Editore', webpage, 'uploader')
             upload_date = unified_strdate(self._html_search_meta(
-                'item-date', webpage, 'upload date'))
+                'item-date', webpage, 'upload date', default=None))
 
         subtitles = self.extract_subtitles(video_id, webpage)
 

From cd47a628fcfb4ec49d0559bdedf792d99a52d53d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 22:41:52 +0600
Subject: [PATCH 0221/1515] [rai] Add test for #5396

---
 youtube_dl/extractor/rai.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 12127c634..1631faf29 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -62,6 +62,17 @@ class RaiIE(InfoExtractor):
                 'description': 'Edizione delle ore 20:30 ',
             }
         },
+        {
+            'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
+            'md5': '02b64456f7cc09f96ff14e7dd489017e',
+            'info_dict': {
+                'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
+                'ext': 'flv',
+                'title': 'Il Candidato - Primo episodio: "Le Primarie"',
+                'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!',
+                'uploader': 'RaiTre',
+            }
+        }
     ]
 
     def _extract_relinker_url(self, webpage):

From 66ee7b3234482ddcb24849d3ab64db382fff5bfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 23:36:28 +0600
Subject: [PATCH 0222/1515] [ted] Extract all formats (Closes #5397)

---
 youtube_dl/extractor/ted.py | 51 +++++++++++++++++++++++++++----------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 4cec06f8b..a2dc14c2b 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -5,9 +5,8 @@ import re
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_str,
-)
+from ..compat import compat_str
+from ..utils import int_or_none
 
 
 class TEDIE(InfoExtractor):
@@ -170,17 +169,41 @@ class TEDIE(InfoExtractor):
                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
                 if finfo:
                     f.update(finfo)
-        else:
-            # Use rtmp downloads
-            formats = [{
-                'format_id': f['name'],
-                'url': talk_info['streamer'],
-                'play_path': f['file'],
-                'ext': 'flv',
-                'width': f['width'],
-                'height': f['height'],
-                'tbr': f['bitrate'],
-            } for f in talk_info['resources']['rtmp']]
+
+        for format_id, resources in talk_info['resources'].items():
+            if format_id == 'h264':
+                for resource in resources:
+                    bitrate = int_or_none(resource.get('bitrate'))
+                    formats.append({
+                        'url': resource['file'],
+                        'format_id': '%s-%sk' % (format_id, bitrate),
+                        'tbr': bitrate,
+                    })
+            elif format_id == 'rtmp':
+                streamer = talk_info.get('streamer')
+                if not streamer:
+                    continue
+                for resource in resources:
+                    formats.append({
+                        'format_id': '%s-%s' % (format_id, resource.get('name')),
+                        'url': streamer,
+                        'play_path': resource['file'],
+                        'ext': 'flv',
+                        'width': int_or_none(resource.get('width')),
+                        'height': int_or_none(resource.get('height')),
+                        'tbr': int_or_none(resource.get('bitrate')),
+                    })
+            elif format_id == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id))
+
+        audio_download = talk_info.get('audioDownload')
+        if audio_download:
+            formats.append({
+                'url': audio_download,
+                'format_id': 'audio',
+            })
+
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])

From 65939effb55087f584ecd5d4b304eadbdef875d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 18:52:41 +0600
Subject: [PATCH 0223/1515] [hitbox:live] Fix hls extration (Closes #5315)

---
 youtube_dl/extractor/hitbox.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 84bd7c080..d606429ca 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -10,6 +10,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     compat_str,
+    determine_ext,
 )
 
 
@@ -147,12 +148,27 @@ class HitboxLiveIE(HitboxIE):
                 servers.append(base_url)
                 for stream in cdn.get('bitrates'):
                     label = stream.get('label')
-                    if label != 'Auto':
+                    if label == 'Auto':
+                        continue
+                    stream_url = stream.get('url')
+                    if not stream_url:
+                        continue
+                    bitrate = int_or_none(stream.get('bitrate'))
+                    if stream.get('provider') == 'hls' or determine_ext(stream_url) == 'm3u8':
+                        if not stream_url.startswith('http'):
+                            continue
                         formats.append({
-                            'url': '%s/%s' % (base_url, stream.get('url')),
+                            'url': stream_url,
                             'ext': 'mp4',
-                            'vbr': stream.get('bitrate'),
-                            'resolution': label,
+                            'tbr': bitrate,
+                            'format_note': label,
+                            'rtmp_live': True,
+                        })
+                    else:
+                        formats.append({
+                            'url': '%s/%s' % (base_url, stream_url),
+                            'ext': 'mp4',
+                            'tbr': bitrate,
                             'rtmp_live': True,
                             'format_note': host,
                             'page_url': url,

From 7a91d1fc43f8873e6636c316006c908d7efe2f07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:03:12 +0600
Subject: [PATCH 0224/1515] [crooksandliars] Improve embed extractor and remove
 article extractor

---
 youtube_dl/extractor/__init__.py       |  2 +-
 youtube_dl/extractor/crooksandliars.py | 87 +++++++++++---------------
 2 files changed, 39 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 894aa5b43..bbf3be41d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,7 +90,7 @@ from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
-from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
+from .crooksandliars import CrooksAndLiarsIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index cee0603f4..143509004 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -1,71 +1,60 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 from ..utils import (
-    mimetype2ext,
+    int_or_none,
+    qualities,
 )
 
 
 class CrooksAndLiarsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
-
+    _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
         'info_dict': {
-            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'id': '8RUoRhRi',
+            'ext': 'mp4',
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
-            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,
-            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
-            'uploader': "Heather",
+            'upload_date': '20150405',
+            'uploader': 'Heather',
+            'duration': 236,
         }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
-
-        formats = []
-        for item in manifest['flavors']:
-            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
-                continue
-            formats.append({
-                'format_id': item['type'],
-                'ext': mimetype2ext(item['mime']),
-                'url': item['url'],
-            })
-
-        # XXX: manifest['url']?
-        return {
-            'url': url,
-            'id': video_id,
-            'uploader': manifest['author'],
-            'title': manifest['title'],
-            'description': manifest['description'],
-            'thumbnail': self._proto_relative_url(manifest['poster']),
-            'duration': manifest['duration'],
-            'timestamp': int(manifest['created']),
-            'formats': formats,
-        }
-
-class CrooksAndLiarsArticleIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
-
-    _TESTS = [{
-        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+    }, {
+        'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+        webpage = self._download_webpage(
+            'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)
+
+        manifest = self._parse_json(
+            self._search_regex(
+                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
+            video_id)
+
+        quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))
+
+        formats = [{
+            'url': item['url'],
+            'format_id': item['type'],
+            'quality': quality(item['type']),
+        } for item in manifest['flavors'] if item['mime'].startswith('video/')]
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url',
-            'url': player_url
+            'url': url,
+            'id': video_id,
+            'title': manifest['title'],
+            'description': manifest.get('description'),
+            'thumbnail': self._proto_relative_url(manifest.get('poster')),
+            'timestamp': int_or_none(manifest.get('created')),
+            'uploader': manifest.get('author'),
+            'duration': int_or_none(manifest.get('duration')),
+            'formats': formats,
         }

From 18153f1b3249701a82c24ec492f5d58929292caf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:20:20 +0600
Subject: [PATCH 0225/1515] [generic] Add support for Crooks and Liars embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6c212efac..0dc52ed23 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1275,6 +1275,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
 
+        # Look for Crooks and Liars embeds
+        mobj = re.search(
+            r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for NBC Sports VPlayer embeds
         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
         if nbc_sports_url:

From a4257017ef2ee665d35dd71905db03ca3913c92e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:26:42 +0600
Subject: [PATCH 0226/1515] [generic] Add tests for Crooks and Liars embeds

---
 youtube_dl/extractor/generic.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0dc52ed23..7ad555e9f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -642,6 +642,32 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
+        # Crooks and Liars embed
+        {
+            'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+            'info_dict': {
+                'id': '8RUoRhRi',
+                'ext': 'mp4',
+                'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+                'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+                'timestamp': 1428207000,
+                'upload_date': '20150405',
+                'uploader': 'Heather',
+            },
+        },
+        # Crooks and Liars external embed
+        {
+            'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
+            'info_dict': {
+                'id': 'MTE3MjUtMzQ2MzA',
+                'ext': 'mp4',
+                'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
+                'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
+                'timestamp': 1265032391,
+                'upload_date': '20100201',
+                'uploader': 'Heather',
+            },
+        },
         # NBC Sports vplayer embed
         {
             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',

From ed5641e2499de9dfbfa78c6684e92e9581402f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:27:39 +0600
Subject: [PATCH 0227/1515] [crooksandliars] Quotes consistency

---
 youtube_dl/extractor/crooksandliars.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index 143509004..443eb7691 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -14,7 +14,7 @@ class CrooksAndLiarsIE(InfoExtractor):
         'info_dict': {
             'id': '8RUoRhRi',
             'ext': 'mp4',
-            'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+            'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
             'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,

From b04b94da5fbe944e2a9d2946ab8b3acb212f9f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 03:57:56 +0600
Subject: [PATCH 0228/1515] [options] Fix file based configurations for python
 2 (Closes #5401)

---
 youtube_dl/options.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5720fb424..11603f60d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -794,21 +794,22 @@ def parseOpts(overrideArguments=None):
         if opts.verbose:
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
-        command_line_conf = sys.argv[1:]
-        # Workaround for Python 2.x, where argv is a byte list
-        if sys.version_info < (3,):
-            command_line_conf = [
-                a.decode(preferredencoding(), 'replace') for a in command_line_conf]
+        def compat_conf(conf):
+            if sys.version_info < (3,):
+                return [a.decode(preferredencoding(), 'replace') for a in conf]
+            return conf
+
+        command_line_conf = compat_conf(sys.argv[1:])
 
         if '--ignore-config' in command_line_conf:
             system_conf = []
             user_conf = []
         else:
-            system_conf = _readOptions('/etc/youtube-dl.conf')
+            system_conf = compat_conf(_readOptions('/etc/youtube-dl.conf'))
             if '--ignore-config' in system_conf:
                 user_conf = []
             else:
-                user_conf = _readUserConf()
+                user_conf = compat_conf(_readUserConf())
         argv = system_conf + user_conf + command_line_conf
 
         opts, args = parser.parse_args(argv)

From 7d2ba6394c489879578e316e533eeb282942e54b Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sun, 12 Apr 2015 11:23:01 +0200
Subject: [PATCH 0229/1515] [FootyRoom] Fixed missing http prefix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For some reason FootyTube is missing the „http:“ prefix on some
Playwire links for some videos
---
 youtube_dl/extractor/footyroom.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 2b4691ae8..dc784db96 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -6,14 +6,25 @@ from .common import InfoExtractor
 
 class FootyRoomIE(InfoExtractor):
     _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
+        'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
             'id': 'schalke-04-0-2-real-madrid-2015-02',
             'title': 'Schalke 04 0 – 2 Real Madrid',
         },
         'playlist_count': 3,
-    }
+    },
+    {
+        'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
+        'info_dict': {
+            'id': 'georgia-0-2-germany-2015-03',
+            'title': 'Georgia 0 – 2 Germany',
+        },
+        'playlist_count': 1,
+    },
+
+    ]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -35,6 +46,8 @@ class FootyRoomIE(InfoExtractor):
             playwire_url = self._search_regex(
                 r'data-config="([^"]+)"', payload,
                 'playwire url', default=None)
+            if not playwire_url.startswith("http:"):
+                playwire_url = "http:" + playwire_url
             if playwire_url:
                 entries.append(self.url_result(playwire_url, 'Playwire'))
 

From 9a4d8fae82f10afe8b2d0611f2f054af60dc7acc Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sun, 12 Apr 2015 11:31:58 +0200
Subject: [PATCH 0230/1515] [FootyTube] Fixed wrong md5 checksum

---
 youtube_dl/extractor/footyroom.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index dc784db96..70dfb0659 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -8,7 +8,6 @@ class FootyRoomIE(InfoExtractor):
     _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
-        'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
             'id': 'schalke-04-0-2-real-madrid-2015-02',
             'title': 'Schalke 04 0 – 2 Real Madrid',

From 504c1cedfeb1562b089fa83ca258966e22acc773 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:09:52 +0600
Subject: [PATCH 0231/1515] [footyroom] Improve

---
 youtube_dl/extractor/footyroom.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 70dfb0659..385a99b13 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -45,9 +45,8 @@ class FootyRoomIE(InfoExtractor):
             playwire_url = self._search_regex(
                 r'data-config="([^"]+)"', payload,
                 'playwire url', default=None)
-            if not playwire_url.startswith("http:"):
-                playwire_url = "http:" + playwire_url
             if playwire_url:
-                entries.append(self.url_result(playwire_url, 'Playwire'))
+                entries.append(self.url_result(self._proto_relative_url(
+                    playwire_url, 'http:'), 'Playwire'))
 
         return self.playlist_result(entries, playlist_id, playlist_title)

From ac58e68bc396904b8e7afa21c9d65a55d2a4852b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:11:11 +0600
Subject: [PATCH 0232/1515] [footyroom] Remove superfluous whitespace

---
 youtube_dl/extractor/footyroom.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 385a99b13..4c7dbca40 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -13,17 +13,14 @@ class FootyRoomIE(InfoExtractor):
             'title': 'Schalke 04 0 – 2 Real Madrid',
         },
         'playlist_count': 3,
-    },
-    {
+    }, {
         'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
         'info_dict': {
             'id': 'georgia-0-2-germany-2015-03',
             'title': 'Georgia 0 – 2 Germany',
         },
         'playlist_count': 1,
-    },
-
-    ]
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)

From e91b2d14e3100801269ec9e5e853fd8a21c29443 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:17:31 +0600
Subject: [PATCH 0233/1515] Credit @snipem for gamersyde (#5352)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9c65dc1d4..db3f42b26 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -122,3 +122,4 @@ Joram Schrijver
 Will W.
 Mohammad Teimori Pabandi
 Roman Le Négrate
+Matthias Küch

From c36a95954947fd5d9fb8df2539115a7f9a6b3a59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:36:29 +0600
Subject: [PATCH 0234/1515] [YoutubeDL] Try to download worst audio+video
 served by a single file first (Closes #5408)

---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 640b8c99d..a68b24ab4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -919,6 +919,11 @@ class YoutubeDL(object):
         if format_spec == 'best' or format_spec is None:
             return available_formats[-1]
         elif format_spec == 'worst':
+            audiovideo_formats = [
+                f for f in available_formats
+                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+            if audiovideo_formats:
+                return audiovideo_formats[0]
             return available_formats[0]
         elif format_spec == 'bestaudio':
             audio_formats = [

From 830d53bfae7a665b55656dd50c9f35f0d0b0161d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 23:11:47 +0600
Subject: [PATCH 0235/1515] [utils] Add `video_title` for `url_result`

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 530c449c1..8ed97f8dd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -492,7 +492,7 @@ class InfoExtractor(object):
 
     # Methods for following #608
     @staticmethod
-    def url_result(url, ie=None, video_id=None):
+    def url_result(url, ie=None, video_id=None, video_title=None):
         """Returns a url that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
@@ -500,6 +500,8 @@ class InfoExtractor(object):
                       'ie_key': ie}
         if video_id is not None:
             video_info['id'] = video_id
+        if video_title is not None:
+            video_info['title'] = video_title
         return video_info
 
     @staticmethod

From fb69240ca0934299583bf6c7a855d5c602a4a7e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 23:19:00 +0600
Subject: [PATCH 0236/1515] [youtube] Extract video titles for channel playlist
 if possible (Closes #4971)

---
 youtube_dl/extractor/youtube.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2774ec30b..791e1fe62 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1370,10 +1370,18 @@ class YoutubeChannelIE(InfoExtractor):
 
     def extract_videos_from_page(self, page):
         ids_in_page = []
-        for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
-            if mobj.group(1) not in ids_in_page:
-                ids_in_page.append(mobj.group(1))
-        return ids_in_page
+        titles_in_page = []
+        for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):
+            video_id = mobj.group('id')
+            video_title = unescapeHTML(mobj.group('title'))
+            try:
+                idx = ids_in_page.index(video_id)
+                if video_title and not titles_in_page[idx]:
+                    titles_in_page[idx] = video_title
+            except ValueError:
+                ids_in_page.append(video_id)
+                titles_in_page.append(video_title)
+        return zip(ids_in_page, titles_in_page)
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
@@ -1390,10 +1398,12 @@ class YoutubeChannelIE(InfoExtractor):
         if autogenerated:
             # The videos are contained in a single page
             # the ajax pages can't be used, they are empty
-            video_ids = self.extract_videos_from_page(channel_page)
+            videos = self.extract_videos_from_page(channel_page)
             entries = [
-                self.url_result(video_id, 'Youtube', video_id=video_id)
-                for video_id in video_ids]
+                self.url_result(
+                    video_id, 'Youtube', video_id=video_id,
+                    video_title=video_title)
+                for video_id, video_title in videos]
             return self.playlist_result(entries, channel_id)
 
         def _entries():
@@ -1401,9 +1411,10 @@ class YoutubeChannelIE(InfoExtractor):
             for pagenum in itertools.count(1):
 
                 ids_in_page = self.extract_videos_from_page(content_html)
-                for video_id in ids_in_page:
+                for video_id, video_title in ids_in_page:
                     yield self.url_result(
-                        video_id, 'Youtube', video_id=video_id)
+                        video_id, 'Youtube', video_id=video_id,
+                        video_title=video_title)
 
                 mobj = re.search(
                     r'data-uix-load-more-href="/?(?P<more>[^"]+)"',

From 7bd930368c48222b0b14211840ac7951fd62f3cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Apr 2015 00:08:39 +0600
Subject: [PATCH 0237/1515] [youtube] Remove unused variable

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 791e1fe62..dc9e15e98 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1386,7 +1386,6 @@ class YoutubeChannelIE(InfoExtractor):
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        video_ids = []
         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
         channel_page = self._download_webpage(url, channel_id)
         autogenerated = re.search(r'''(?x)

From 51f1244600beb8c3182dc0df756f61cbcfb6c13c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 13 Apr 2015 19:26:15 +0800
Subject: [PATCH 0238/1515] [vine] flake8

---
 youtube_dl/extractor/vine.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d4f5a991e..65c459fad 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor

From 8f02ad4f12549865a2a4436328075f4b20b906ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Apr 2015 20:28:16 +0600
Subject: [PATCH 0239/1515] [youtube] Simplify

---
 youtube_dl/extractor/youtube.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc9e15e98..52909b0da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1397,20 +1397,18 @@ class YoutubeChannelIE(InfoExtractor):
         if autogenerated:
             # The videos are contained in a single page
             # the ajax pages can't be used, they are empty
-            videos = self.extract_videos_from_page(channel_page)
             entries = [
                 self.url_result(
                     video_id, 'Youtube', video_id=video_id,
                     video_title=video_title)
-                for video_id, video_title in videos]
+                for video_id, video_title in self.extract_videos_from_page(channel_page)]
             return self.playlist_result(entries, channel_id)
 
         def _entries():
             more_widget_html = content_html = channel_page
             for pagenum in itertools.count(1):
 
-                ids_in_page = self.extract_videos_from_page(content_html)
-                for video_id, video_title in ids_in_page:
+                for video_id, video_title in self.extract_videos_from_page(content_html):
                     yield self.url_result(
                         video_id, 'Youtube', video_id=video_id,
                         video_title=video_title)

From 37b44fe7c1e462b748171c4af743f30c01c95fe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 13 Apr 2015 22:50:40 +0200
Subject: [PATCH 0240/1515] [postprocessor/atomicparsley] Don't try to remove
 the temporary and original files if the format is unsupported (fixes #5419)

---
 youtube_dl/postprocessor/atomicparsley.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
index 448ccc5f3..a5dfc136a 100644
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ b/youtube_dl/postprocessor/atomicparsley.py
@@ -50,8 +50,13 @@ class AtomicParsleyPP(PostProcessor):
             msg = stderr.decode('utf-8', 'replace').strip()
             raise AtomicParsleyPPError(msg)
 
-        os.remove(encodeFilename(filename))
         os.remove(encodeFilename(temp_thumbnail))
-        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+        # for formats that don't support thumbnails (like 3gp) AtomicParsley
+        # won't create to the temporary file
+        if b'No changes' in stdout:
+            self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
+        else:
+            os.remove(encodeFilename(filename))
+            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info

From edfcf7abe2f31bb8309ca032e738c90dc3a5722e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Apr 2015 12:45:43 +0800
Subject: [PATCH 0241/1515] [generic] Support another type of Ooyala embedded
 video

---
 youtube_dl/extractor/generic.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7ad555e9f..92a62e866 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -688,6 +688,20 @@ class GenericIE(InfoExtractor):
                 'title': '生物老師男變女 全校挺"做自己"',
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
+        },
+        # Ooyala embed
+        {
+            'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
+            'info_dict': {
+                'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
+                'ext': 'mp4',
+                'description': 'VIDEO: Index/Match versus VLOOKUP.',
+                'title': 'This is what separates the Excel masters from the wannabes',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
         }
     ]
 
@@ -1092,7 +1106,8 @@ class GenericIE(InfoExtractor):
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
-                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage))
+                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
+                re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 

From 01c58f84738e056733717174d1076ec465c62500 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Apr 2015 13:10:10 +0800
Subject: [PATCH 0242/1515] [generic] Fix test generic_51

The website replaced the original video with a new one
---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 92a62e866..eaf9c769a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -681,11 +681,11 @@ class GenericIE(InfoExtractor):
         # UDN embed
         {
             'url': 'http://www.udn.com/news/story/7314/822787',
-            'md5': 'de06b4c90b042c128395a88f0384817e',
+            'md5': 'fd2060e988c326991037b9aff9df21a6',
             'info_dict': {
-                'id': '300040',
+                'id': '300346',
                 'ext': 'mp4',
-                'title': '生物老師男變女 全校挺"做自己"',
+                'title': '中一中男師變性 全校師生力挺',
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
         },

From 8da1bb04186d4147f89923abc09a9db0fa2a4fec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Apr 2015 15:27:56 +0800
Subject: [PATCH 0243/1515] [miomio] Enhance error checking and replace dead
 test case

---
 youtube_dl/extractor/miomio.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index cc3f27194..d41195a96 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     xpath_text,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -14,13 +15,14 @@ class MioMioIE(InfoExtractor):
     IE_NAME = 'miomio.tv'
     _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://www.miomio.tv/watch/cc179734/',
-        'md5': '48de02137d0739c15b440a224ad364b9',
+        # "type=video" in flashvars
+        'url': 'http://www.miomio.tv/watch/cc88912/',
+        'md5': '317a5f7f6b544ce8419b784ca8edae65',
         'info_dict': {
-            'id': '179734',
+            'id': '88912',
             'ext': 'flv',
-            'title': '手绘动漫鬼泣但丁全程画法',
-            'duration': 354,
+            'title': '【SKY】字幕 铠武昭和VS平成 假面骑士大战FEAT战队 魔星字幕组 字幕',
+            'duration': 5923,
         },
     }, {
         'url': 'http://www.miomio.tv/watch/cc184024/',
@@ -42,7 +44,7 @@ class MioMioIE(InfoExtractor):
             r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
 
         xml_config = self._search_regex(
-            r'flashvars="type=sina&amp;(.+?)&amp;',
+            r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
             webpage, 'xml config')
 
         # skipping the following page causes lags and eventually connection drop-outs
@@ -59,6 +61,9 @@ class MioMioIE(InfoExtractor):
             'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
         }
 
+        if not int_or_none(xpath_text(vid_config, 'timelength')):
+            raise ExtractorError('Unable to load videos!', expected=True)
+
         entries = []
         for f in vid_config.findall('./durl'):
             segment_url = xpath_text(f, 'url', 'video url')

From 476e1095fac398b648f953a826ed3b191ce8d5d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Apr 2015 17:48:41 +0600
Subject: [PATCH 0244/1515] [brightcove] Improve brightcove experience regex
 (Closes #5421)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0733bece7..b37857b2e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -183,7 +183,7 @@ class BrightcoveIE(InfoExtractor):
             (?:
                 [^>]+?class=[\'"][^>]*?BrightcoveExperience.*?[\'"] |
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
-            ).+?</object>''',
+            ).+?>\s*</object>''',
             webpage)
         return [cls._build_brighcove_url(m) for m in matches]
 

From 94c1255782414cdcda565717e31c91734ec00990 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Apr 2015 17:50:53 +0600
Subject: [PATCH 0245/1515] [brightcove] Handle non well-formed XMLs (#5421)

---
 youtube_dl/extractor/brightcove.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b37857b2e..117cb00e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -117,7 +117,10 @@ class BrightcoveIE(InfoExtractor):
         object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
         object_str = fix_xml_ampersands(object_str)
 
-        object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+        try:
+            object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+        except xml.etree.ElementTree.ParseError:
+            return
 
         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')
         if fv_el is not None:
@@ -185,7 +188,7 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return [cls._build_brighcove_url(m) for m in matches]
+        return filter(None, [cls._build_brighcove_url(m) for m in matches])
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From ae849ca1701dc5bdf25e648a57a1e6ec64f2f968 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 15 Apr 2015 17:45:28 +0800
Subject: [PATCH 0246/1515] [tumblr] Dismiss warnings for optional fields
 (fixes #5202)

---
 youtube_dl/extractor/tumblr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 2a1ae5a71..828c808a6 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -56,6 +56,6 @@ class TumblrIE(InfoExtractor):
             'url': video_url,
             'ext': 'mp4',
             'title': video_title,
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
         }

From 9c5335a0270042b7e84f02e550676447ccf603b1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 15 Apr 2015 19:56:21 +0800
Subject: [PATCH 0247/1515] [teamcoco] Fix "preload" data extraction (fixes
 #5179)

---
 youtube_dl/extractor/teamcoco.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 1caf08cb7..2381676b4 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import base64
@@ -35,6 +36,17 @@ class TeamcocoIE(InfoExtractor):
                 'duration': 288,
                 'age_limit': 0,
             }
+        }, {
+            'url': 'http://teamcoco.com/video/timothy-olyphant-drinking-whiskey',
+            'info_dict': {
+                'id': '88748',
+                'ext': 'mp4',
+                'title': 'Timothy Olyphant Raises A Toast To “Justified”',
+                'description': 'md5:15501f23f020e793aeca761205e42c24',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 downloads
+            }
         }
     ]
     _VIDEO_ID_REGEXES = (
@@ -54,10 +66,23 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
+        preload = None
         preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
-        if not preloads:
-            raise ExtractorError('Preload information could not be extracted')
-        preload = max([(len(p), p) for p in preloads])[1]
+        if preloads:
+            preload = max([(len(p), p) for p in preloads])[1]
+
+        if not preload:
+            preload = ''.join(re.findall(r'this\.push\("([^"]+)"\);', webpage))
+
+        if not preload:
+            preload = self._html_search_regex([
+                r'player,\[?"([^"]+)"\]?', r'player.init\(\[?"([^"]+)"\]?\)'
+            ], webpage.replace('","', ''), 'preload data', default=None)
+
+        if not preload:
+            raise ExtractorError(
+                'Preload information could not be extracted', expected=True)
+
         data = self._parse_json(
             base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 

From 4d1cdb5bfebcc93d73ffb710ec7af0752d698dca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 20:58:48 +0600
Subject: [PATCH 0248/1515] [spike] Extend _VALID_URL (Closes #5420)

---
 youtube_dl/extractor/spike.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index e529bb55c..182f286df 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -5,7 +5,7 @@ from .mtv import MTVServicesInfoExtractor
 
 class SpikeIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://
-        (?:www\.spike\.com/(?:video-clips|(?:full-)?episodes)/.+|
+        (?:www\.spike\.com/(?:video-(?:clips|playlists)|(?:full-)?episodes)/.+|
          m\.spike\.com/videos/video\.rbml\?id=(?P<id>[^&]+))
         '''
     _TEST = {

From 0dfe9bc9d286ceb0fd35b0fc8857ba1a86cecdd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 21:02:34 +0600
Subject: [PATCH 0249/1515] [mtv] Capture and output error message (#5420)

---
 youtube_dl/extractor/mtv.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index c11de1cb6..4430b3416 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -118,6 +118,14 @@ class MTVServicesInfoExtractor(InfoExtractor):
         mediagen_doc = self._download_xml(mediagen_url, video_id,
                                           'Downloading video urls')
 
+        item = mediagen_doc.find('./video/item')
+        if item is not None and item.get('type') == 'text':
+            message = '%s returned error: ' % self.IE_NAME
+            if item.get('code') is not None:
+                message += '%s - ' % item.get('code')
+            message += item.text
+            raise ExtractorError(message, expected=True)
+
         description_node = itemdoc.find('description')
         if description_node is not None:
             description = description_node.text.strip()

From 2dcc114f84ebd6ea18b60a57070bcb81861d8f84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 22:10:08 +0600
Subject: [PATCH 0250/1515] [generic] Add support for playwire embeds (Closes
 #5430)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index eaf9c769a..ab3538694 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1310,6 +1310,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Pladform')
 
+        # Look for Playwire embeds
+        mobj = re.search(
+            r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for 5min embeds
         mobj = re.search(
             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)

From c798f15b989bc8c3578c5b0baf75f4fb4760ba81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 22:13:01 +0600
Subject: [PATCH 0251/1515] [generic] Add test for playwire embed (#5430)

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ab3538694..7f2faa935 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -622,6 +622,17 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
+        # Playwire embed
+        {
+            'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
+            'info_dict': {
+                'id': '3519514',
+                'ext': 'mp4',
+                'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
+                'thumbnail': 're:^https?://.*\.png$',
+                'duration': 45.115,
+            },
+        },
         # 5min embed
         {
             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',

From 9fc03aa87c74cb199f9e1e146ac5cb71b166d3e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 00:27:39 +0800
Subject: [PATCH 0252/1515] [brightcove] Always return lists from
 _extract_brightcove_urls

In Python 3, filter() returns an iterable object, which is equivalently
to True even for an empty result set. It causes false positive playlists
in generic extraction logic.
---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 117cb00e6..4f60d5366 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -188,7 +188,7 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return filter(None, [cls._build_brighcove_url(m) for m in matches])
+        return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From afe4a8c7699e70ecaa7f273b55a038b4d8a1050f Mon Sep 17 00:00:00 2001
From: FireDart <firedartonline@gmail.com>
Date: Wed, 15 Apr 2015 22:17:45 -0400
Subject: [PATCH 0253/1515] [gfycat] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/gfycat.py   | 107 +++++++++++++++++++++++++++++++
 2 files changed, 108 insertions(+)
 create mode 100644 youtube_dl/extractor/gfycat.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..d32f1cbd2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -185,6 +185,7 @@ from .gametrailers import GametrailersIE
 from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
+from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
new file mode 100644
index 000000000..d103693cd
--- /dev/null
+++ b/youtube_dl/extractor/gfycat.py
@@ -0,0 +1,107 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import datetime
+
+from .common import InfoExtractor
+
+class GfycatIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
+        'info_dict': {
+            'id':          'DeadlyDecisiveGermanpinscher',
+            'title':       'Ghost in the Shell',
+            'ext':         'mp4',
+            'upload_date': '20140913'
+        }
+    },{
+        'url': 'http://gfycat.com/pleasinghilariouskusimanse',
+        'info_dict': {
+            'id':          'pleasinghilariouskusimanse',
+            'title':       'PleasingHilariousKusimanse',
+            'ext':         'webm',
+            'upload_date': '20150412'
+        }
+    },{
+        'url': 'http://gfycat.com/requiredunkemptbuzzard',
+        'info_dict': {
+            'id':          'requiredunkemptbuzzard',
+            'title':       'Headshot!',
+            'ext':         'gif',
+            'upload_date': '20150130'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json     = self._download_json("http://gfycat.com/cajax/get/" + video_id, video_id, 'Downloading video info')['gfyItem']
+        
+        # Title
+        # Use user title first, else fallback to url formated name
+        if json['title']:
+            video_title = json['title']
+        else:
+            video_title = json['gfyName']
+        
+        # Formats
+        # Pref: mp4, webm, gif
+        formats = [{
+            'format_id':  'mp4',
+            'ext':        'mp4',
+            'url':        json['mp4Url'],
+            'width':      json['width'],
+            'height':     json['height'],
+            'fps':        json['frameRate'],
+            'filesize':   json['mp4Size'],
+            'preference': '-1'
+        }, {
+            'format_id': 'webm',
+            'ext':       'webm',
+            'url':        json['webmUrl'],
+            'width':      json['width'],
+            'height':     json['height'],
+            'fps':        json['frameRate'],
+            'filesize':   json['webmSize'],
+            'preference': 0
+        }, {
+            'format_id':  'gif',
+            'ext':        'gif',
+            'url':        json['gifUrl'],
+            'width':      json['width'],
+            'height':     json['height'],
+            'fps':        json['frameRate'],
+            'filesize':   json['gifSize'],
+            'preference': 1
+        }]
+        
+        self._sort_formats(formats)
+        
+        # Date
+        date = datetime.datetime.fromtimestamp(
+            int(json['createDate'])
+        ).strftime('%Y%m%d')
+        
+        # Length
+        duration = json['numFrames'] / json['frameRate']
+        
+        # Age limit
+        # 1 = nsfw / 0 = sfw
+        if json['nsfw'] == 1:
+            age_limit = 18
+        else:
+            age_limit = 0
+        
+        return {
+            'id':          video_id,
+            'title':       video_title,
+            'formats':     formats,
+            'creator':     json['userName'],
+            'description': json['description'],
+            'upload_date': date,
+            'categories':  json['tags'],
+            'age_limit':   age_limit,
+            'duration':    duration,
+            'view_count':  json['views']
+        }

From d0eb724e22dc2e48f206dac45f9db9c17dcb26e1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 17:04:53 +0800
Subject: [PATCH 0254/1515] [UDNEmbed] Enhance error checking and extend
 _VALID_URL

---
 youtube_dl/extractor/udn.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index bba25bb58..c08428acf 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -3,12 +3,15 @@ from __future__ import unicode_literals
 
 import json
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    js_to_json,
+    ExtractorError,
+)
 from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//video\.udn\.com/embed/news/(?P<id>\d+)'
+    _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',
         'md5': 'de06b4c90b042c128395a88f0384817e',
@@ -19,7 +22,11 @@ class UDNEmbedIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }, {
-        'url': '//video.udn.com/embed/news/300040',
+        'url': 'https://video.udn.com/embed/news/300040',
+        'only_matching': True,
+    }, {
+        # From https://video.udn.com/news/303776
+        'url': 'https://video.udn.com/play/news/303776',
         'only_matching': True,
     }]
 
@@ -47,7 +54,10 @@ class UDNEmbedIE(InfoExtractor):
                 'retrieve url for %s video' % video_type),
             'format_id': video_type,
             'preference': 0 if video_type == 'mp4' else -1,
-        } for video_type, api_url in video_urls.items()]
+        } for video_type, api_url in video_urls.items() if api_url]
+
+        if not formats:
+            raise ExtractorError('No videos found', expected=True)
 
         self._sort_formats(formats)
 

From d6fd958c5f6847f39f4ed653e82832f2f44657ba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 17:16:11 +0800
Subject: [PATCH 0255/1515] [generic] Extract videos from SMIL manifests
 (closes #5145 and fixes #5135)

---
 youtube_dl/extractor/generic.py | 37 ++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7f2faa935..8c859f068 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -713,6 +713,20 @@ class GenericIE(InfoExtractor):
                 # m3u8 downloads
                 'skip_download': True,
             }
+        },
+        # Contains a SMIL manifest
+        {
+            'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
+            'info_dict': {
+                'id': 'file',
+                'ext': 'flv',
+                'title': '+ Football: Lottery Champions League Europe',
+                'uploader': 'www.telewebion.com',
+            },
+            'params': {
+                # rtmpe downloads
+                'skip_download': True,
+            }
         }
     ]
 
@@ -1440,13 +1454,22 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
-            entries.append({
-                'id': video_id,
-                'url': video_url,
-                'uploader': video_uploader,
-                'title': video_title,
-                'age_limit': age_limit,
-            })
+            if determine_ext(video_url) == 'smil':
+                entries.append({
+                    'id': video_id,
+                    'formats': self._extract_smil_formats(video_url, video_id),
+                    'uploader': video_uploader,
+                    'title': video_title,
+                    'age_limit': age_limit,
+                })
+            else:
+                entries.append({
+                    'id': video_id,
+                    'url': video_url,
+                    'uploader': video_uploader,
+                    'title': video_title,
+                    'age_limit': age_limit,
+                })
 
         if len(entries) == 1:
             return entries[0]

From 5d98908b26a4bf593abd6340d34d983f8d0980df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 17 Mar 2015 19:03:29 +0800
Subject: [PATCH 0256/1515] [QQMusic] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/qqmusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..63bcf666f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -396,6 +396,7 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
+from .qqmusic import QQMusicIE
 from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
new file mode 100644
index 000000000..3dc637392
--- /dev/null
+++ b/youtube_dl/extractor/qqmusic.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import strip_jsonp
+
+# guid is a random number generated in javascript, but seems a fixed number
+# also works
+guid = '1'
+
+
+class QQMusicIE(InfoExtractor):
+    _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
+        'md5': 'bed90b6db2a7a7a7e11bc585f471f63a',
+        'info_dict': {
+            'id': '004295Et37taLD',
+            'ext': 'm4a',
+            'title': '可惜没如果',
+            'upload_date': '20141227',
+            'creator': '林俊杰',
+        }
+    }]
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+
+        detail_info_page = self._download_webpage(
+            'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
+            mid, note='Download sont detail info',
+            errnote='Unable to get song detail info')
+
+        song_name = self._html_search_regex(
+            r"songname:\s*'([^']+)'", detail_info_page, 'song name')
+
+        publish_time = self._html_search_regex(
+            r'发行时间：(\d{4}-\d{2}-\d{2})', detail_info_page,
+            'publish time').replace('-', '')
+
+        singer = self._html_search_regex(
+            r"singer:\s*'([^']+)", detail_info_page, 'singer')
+
+        vkey = self._download_json(
+            'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
+            mid, note='Retrieve vkey', errnote='Unable to get vkey',
+            transform_source=strip_jsonp)['key']
+        song_url = 'http://cc.stream.qqmusic.qq.com/C200%s.m4a?vkey=%s&guid=%s&fromtag=0' % (mid, vkey, guid)
+
+        return {
+            'id': mid,
+            'url': song_url,
+            'title': song_name,
+            'upload_date': publish_time,
+            'creator': singer,
+        }

From a2043572aa99c7517c84c31fe8a7e2051d27bf09 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 18 Mar 2015 13:56:02 +0800
Subject: [PATCH 0257/1515] [QQMusic] Implement the guid algorithm

---
 youtube_dl/extractor/qqmusic.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 3dc637392..93440b954 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -1,13 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
+import time
+
 from .common import InfoExtractor
 from ..utils import strip_jsonp
 
-# guid is a random number generated in javascript, but seems a fixed number
-# also works
-guid = '1'
-
 
 class QQMusicIE(InfoExtractor):
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
@@ -23,6 +22,13 @@ class QQMusicIE(InfoExtractor):
         }
     }]
 
+    # Reference: m_r_GetRUin() in top_player.js
+    # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
+    @staticmethod
+    def m_r_get_ruin():
+        curMs = int(time.time() * 1000) % 1000
+        return int(round(random.random() * 2147483647) * curMs % 1E10)
+
     def _real_extract(self, url):
         mid = self._match_id(url)
 
@@ -41,6 +47,8 @@ class QQMusicIE(InfoExtractor):
         singer = self._html_search_regex(
             r"singer:\s*'([^']+)", detail_info_page, 'singer')
 
+        guid = self.m_r_get_ruin()
+
         vkey = self._download_json(
             'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
             mid, note='Retrieve vkey', errnote='Unable to get vkey',

From 8afff9f849e43c9c4194e302c06b5bec38edd83f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 18 Mar 2015 14:59:33 +0800
Subject: [PATCH 0258/1515] [QQMusic] Add singer info extractor

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/qqmusic.py  | 53 +++++++++++++++++++++++++++++++-
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 63bcf666f..65dbcb6cf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -396,7 +396,10 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
-from .qqmusic import QQMusicIE
+from .qqmusic import (
+    QQMusicIE,
+    QQMusicSingerIE
+)
 from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 93440b954..1c22943a5 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 
 import random
 import time
+import re
 
 from .common import InfoExtractor
 from ..utils import strip_jsonp
+from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -34,7 +36,7 @@ class QQMusicIE(InfoExtractor):
 
         detail_info_page = self._download_webpage(
             'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
-            mid, note='Download sont detail info',
+            mid, note='Download song detail info',
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
@@ -62,3 +64,52 @@ class QQMusicIE(InfoExtractor):
             'upload_date': publish_time,
             'creator': singer,
         }
+
+
+class QQMusicSingerIE(InfoExtractor):
+    _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
+    _TEST = {
+        'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
+        'info_dict': {
+            'id': '001BLpXF2DyJe2',
+            'title': '林俊杰',
+            'description': 'md5:2a222d89ba4455a3af19940c0481bb78',
+        },
+        'playlist_count': 12,
+    }
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+
+        singer_page = self._download_webpage(
+            'http://y.qq.com/y/static/singer/%s/%s/%s.html' % (mid[-2], mid[-1], mid),
+            'Download singer page')
+
+        entries = []
+
+        for item in re.findall(r'<span class="data">([^<>]+)</span>', singer_page):
+            song_mid = item.split('|')[-5]
+            entries.append(self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic', song_mid))
+
+        singer_name = self._html_search_regex(
+            r"singername\s*:\s*'([^']+)'", singer_page, 'singer name',
+            default=None)
+
+        singer_id = self._html_search_regex(
+            r"singerid\s*:\s*'([0-9]+)'", singer_page, 'singer id',
+            default=None)
+
+        singer_desc = None
+
+        if singer_id:
+            req = compat_urllib_request.Request(
+                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
+            req.add_header(
+                'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
+            singer_desc_page = self._download_xml(
+                req, 'Donwload singer description XML')
+
+            singer_desc = singer_desc_page.find('./data/info/desc').text
+
+        return self.playlist_result(entries, mid, singer_name, singer_desc)

From 5edea45fab73874c269655b4cf08da0bbc5ea479 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 19 Mar 2015 01:47:07 +0800
Subject: [PATCH 0259/1515] [QQMusic] Add album info extractor

---
 youtube_dl/extractor/__init__.py |  3 +-
 youtube_dl/extractor/qqmusic.py  | 72 +++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 65dbcb6cf..36860f72c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -398,7 +398,8 @@ from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
 from .qqmusic import (
     QQMusicIE,
-    QQMusicSingerIE
+    QQMusicSingerIE,
+    QQMusicAlbumIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 1c22943a5..d0ea4a769 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -6,7 +6,10 @@ import time
 import re
 
 from .common import InfoExtractor
-from ..utils import strip_jsonp
+from ..utils import (
+    strip_jsonp,
+    unescapeHTML,
+)
 from ..compat import compat_urllib_request
 
 
@@ -66,7 +69,28 @@ class QQMusicIE(InfoExtractor):
         }
 
 
-class QQMusicSingerIE(InfoExtractor):
+class QQPlaylistBaseIE(InfoExtractor):
+    @staticmethod
+    def qq_static_url(category, mid):
+        return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)
+
+    @staticmethod
+    def qq_song_url(mid):
+        return 'http://y.qq.com/#type=song&mid=%s' % mid
+
+    @classmethod
+    def get_entries_from_page(cls, page):
+        entries = []
+
+        for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
+            song_mid = unescapeHTML(item).split('|')[-5]
+            entries.append(cls.url_result(
+                cls.qq_song_url(song_mid), 'QQMusic', song_mid))
+
+        return entries
+
+
+class QQMusicSingerIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -82,15 +106,9 @@ class QQMusicSingerIE(InfoExtractor):
         mid = self._match_id(url)
 
         singer_page = self._download_webpage(
-            'http://y.qq.com/y/static/singer/%s/%s/%s.html' % (mid[-2], mid[-1], mid),
-            'Download singer page')
+            self.qq_static_url('singer', mid), mid, 'Download singer page')
 
-        entries = []
-
-        for item in re.findall(r'<span class="data">([^<>]+)</span>', singer_page):
-            song_mid = item.split('|')[-5]
-            entries.append(self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic', song_mid))
+        entries = self.get_entries_from_page(singer_page)
 
         singer_name = self._html_search_regex(
             r"singername\s*:\s*'([^']+)'", singer_page, 'singer name',
@@ -108,8 +126,40 @@ class QQMusicSingerIE(InfoExtractor):
             req.add_header(
                 'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
             singer_desc_page = self._download_xml(
-                req, 'Donwload singer description XML')
+                req, mid, 'Donwload singer description XML')
 
             singer_desc = singer_desc_page.find('./data/info/desc').text
 
         return self.playlist_result(entries, mid, singer_name, singer_desc)
+
+
+class QQMusicAlbumIE(QQPlaylistBaseIE):
+    _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
+
+    _TEST = {
+        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1&play=0',
+        'info_dict': {
+            'id': '000gXCTb2AhRR1',
+            'title': '我们都是这样长大的',
+            'description': 'md5:d216c55a2d4b3537fe4415b8767d74d6',
+        },
+        'playlist_count': 4,
+    }
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+
+        album_page = self._download_webpage(
+            self.qq_static_url('album', mid), mid, 'Download album page')
+
+        entries = self.get_entries_from_page(album_page)
+
+        album_name = self._html_search_regex(
+            r"albumname\s*:\s*'([^']+)',", album_page, 'album name',
+            default=None)
+
+        album_detail = self._html_search_regex(
+            r'<div class="album_detail close_detail">\s*<p>((?:[^<>]+(?:<br />)?)+)</p>',
+            album_page, 'album details', default=None)
+
+        return self.playlist_result(entries, mid, album_name, album_detail)

From a685ae511a7f2148afb765d0a30bca42a75cf861 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Mar 2015 11:38:53 +0800
Subject: [PATCH 0260/1515] [QQMusic] Song extractor: Add lyrics as description

Note: Test fails on python 3 due to encoding issues
---
 youtube_dl/extractor/qqmusic.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index d0ea4a769..e8aacbc3d 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -24,6 +24,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
+            'description': 'md5:242c97c2847e0495583b7b13764f7106',
         }
     }]
 
@@ -47,10 +48,16 @@ class QQMusicIE(InfoExtractor):
 
         publish_time = self._html_search_regex(
             r'发行时间：(\d{4}-\d{2}-\d{2})', detail_info_page,
-            'publish time').replace('-', '')
+            'publish time', default=None)
+        if publish_time:
+            publish_time = publish_time.replace('-', '')
 
         singer = self._html_search_regex(
-            r"singer:\s*'([^']+)", detail_info_page, 'singer')
+            r"singer:\s*'([^']+)", detail_info_page, 'singer', default=None)
+
+        lrc_content = self._html_search_regex(
+            r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)</div>',
+            detail_info_page, 'LRC lyrics', default=None)
 
         guid = self.m_r_get_ruin()
 
@@ -66,6 +73,7 @@ class QQMusicIE(InfoExtractor):
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,
+            'description': lrc_content,
         }
 
 
@@ -74,10 +82,6 @@ class QQPlaylistBaseIE(InfoExtractor):
     def qq_static_url(category, mid):
         return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)
 
-    @staticmethod
-    def qq_song_url(mid):
-        return 'http://y.qq.com/#type=song&mid=%s' % mid
-
     @classmethod
     def get_entries_from_page(cls, page):
         entries = []
@@ -85,7 +89,8 @@ class QQPlaylistBaseIE(InfoExtractor):
         for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
             song_mid = unescapeHTML(item).split('|')[-5]
             entries.append(cls.url_result(
-                cls.qq_song_url(song_mid), 'QQMusic', song_mid))
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                song_mid))
 
         return entries
 

From c9a779695da56c1f9561af8586ecc586103dc254 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Mar 2015 12:21:27 +0800
Subject: [PATCH 0261/1515] [extractor/common] Add the encoding parameter

The QQMusic info extractor need forced encoding for correct working.
---
 youtube_dl/extractor/common.py  | 34 ++++++++++++++++++++-------------
 youtube_dl/extractor/qqmusic.py |  4 ++--
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ed97f8dd..28f672e42 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -324,7 +324,7 @@ class InfoExtractor(object):
                 self._downloader.report_warning(errmsg)
                 return False
 
-    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
@@ -334,14 +334,11 @@ class InfoExtractor(object):
         if urlh is False:
             assert not fatal
             return False
-        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
+        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
         return (content, urlh)
 
-    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
-        content_type = urlh.headers.get('Content-Type', '')
-        webpage_bytes = urlh.read()
-        if prefix is not None:
-            webpage_bytes = prefix + webpage_bytes
+    @staticmethod
+    def _guess_encoding_from_content(content_type, webpage_bytes):
         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
         if m:
             encoding = m.group(1)
@@ -354,6 +351,16 @@ class InfoExtractor(object):
                 encoding = 'utf-16'
             else:
                 encoding = 'utf-8'
+
+        return encoding
+
+    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
+        content_type = urlh.headers.get('Content-Type', '')
+        webpage_bytes = urlh.read()
+        if prefix is not None:
+            webpage_bytes = prefix + webpage_bytes
+        if not encoding:
+            encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
         if self._downloader.params.get('dump_intermediate_pages', False):
             try:
                 url = url_or_request.get_full_url()
@@ -410,13 +417,13 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
         """ Returns the data of the page as a string """
         success = False
         try_count = 0
         while success is False:
             try:
-                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
                 success = True
             except compat_http_client.IncompleteRead as e:
                 try_count += 1
@@ -431,10 +438,10 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
-                      transform_source=None, fatal=True):
+                      transform_source=None, fatal=True, encoding=None):
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal)
+            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
         if xml_string is False:
             return xml_string
         if transform_source:
@@ -445,9 +452,10 @@ class InfoExtractor(object):
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,
-                       fatal=True):
+                       fatal=True, encoding=None):
         json_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal)
+            url_or_request, video_id, note, errnote, fatal=fatal,
+            encoding=encoding)
         if (not fatal) and json_string is False:
             return None
         return self._parse_json(
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e8aacbc3d..174c8e0ae 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -24,7 +24,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
-            'description': 'md5:242c97c2847e0495583b7b13764f7106',
+            'description': 'md5:4348ff1dd24036906baa7b6f973f8d30',
         }
     }]
 
@@ -41,7 +41,7 @@ class QQMusicIE(InfoExtractor):
         detail_info_page = self._download_webpage(
             'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
             mid, note='Download song detail info',
-            errnote='Unable to get song detail info')
+            errnote='Unable to get song detail info', encoding='gbk')
 
         song_name = self._html_search_regex(
             r"songname:\s*'([^']+)'", detail_info_page, 'song name')

From c052ce6cde53f25949c21d00e4243669f523c0a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 22:00:45 +0800
Subject: [PATCH 0262/1515] [Srf] Add new extractor (fixes #981)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/srf.py      | 86 ++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 youtube_dl/extractor/srf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 36860f72c..830090346 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -485,6 +485,7 @@ from .spike import SpikeIE
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
+from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
new file mode 100644
index 000000000..87b4a676a
--- /dev/null
+++ b/youtube_dl/extractor/srf.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class SrfIE(InfoExtractor):
+    _VALID_URL = r'http://www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=(?P<id>[0-9a-f\-]{36})'
+    _TESTS = [{
+        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'md5': '4cd93523723beff51bb4bee974ee238d',
+        'info_dict': {
+            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+            'display_id': 'snowden-beantragt-asyl-in-russland',
+            'ext': 'm4v',
+            'upload_date': '20130701',
+            'title': 'Snowden beantragt Asyl in Russland',
+            'timestamp': 1372713995,
+        }
+    }, {
+        # No Speichern (Save) button
+        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
+        'info_dict': {
+            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
+            'display_id': 'jaguar-xk120-shadow-und-tornado-dampflokomotive',
+            'ext': 'mp4',
+            'upload_date': '20130710',
+            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
+            'timestamp': 1373493600,
+        },
+        'params': {
+            # Require ffmpeg/avconv
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video_data = self._download_xml(
+            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
+            video_id)
+
+        display_id = re.match(self._VALID_URL, url).group('display_id')
+        title = xpath_text(
+            video_data, './AssetMetadatas/AssetMetadata/title', fatal=True)
+        thumbnails = [{
+            'url': s.text
+        } for s in video_data.findall('.//ImageRepresentation/url')]
+        timestamp = parse_iso8601(xpath_text(video_data, './createdDate'))
+        # The <duration> field in XML is different from the exact duration, skipping
+
+        formats = []
+        for item in video_data.findall('./Playlists/Playlist') + video_data.findall('./Downloads/Download'):
+            url_node = item.find('url')
+            quality = url_node.attrib['quality']
+            full_url = url_node.text
+            original_ext = determine_ext(full_url)
+            if original_ext == 'f4m':
+                full_url += '?hdcore=3.4.0'  # Without this, you get a 403 error
+            formats.append({
+                'url': full_url,
+                'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
+                'format_id': '%s-%s' % (quality, item.attrib['protocol']),
+                'preference': 0 if 'HD' in quality else -1,
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            'title': title,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+        }

From 355c524bfaec10aa8f5ff31bc95d2c50ef1a3113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 20:31:02 +0600
Subject: [PATCH 0263/1515] [srf] Extract all formats and prefer direct links
 over hls and hds

---
 youtube_dl/extractor/srf.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 87b4a676a..4be329a2f 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -61,18 +61,25 @@ class SrfIE(InfoExtractor):
 
         formats = []
         for item in video_data.findall('./Playlists/Playlist') + video_data.findall('./Downloads/Download'):
-            url_node = item.find('url')
-            quality = url_node.attrib['quality']
-            full_url = url_node.text
-            original_ext = determine_ext(full_url)
-            if original_ext == 'f4m':
-                full_url += '?hdcore=3.4.0'  # Without this, you get a 403 error
-            formats.append({
-                'url': full_url,
-                'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
-                'format_id': '%s-%s' % (quality, item.attrib['protocol']),
-                'preference': 0 if 'HD' in quality else -1,
-            })
+            for url_node in item.findall('url'):
+                quality = url_node.attrib['quality']
+                full_url = url_node.text
+                original_ext = determine_ext(full_url)
+                format_id = '%s-%s' % (quality, item.attrib['protocol'])
+                if original_ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        full_url + '?hdcore=3.4.0', video_id, f4m_id=format_id))
+                elif original_ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        full_url, video_id, 'mp4', m3u8_id=format_id))
+                else:
+                    formats.append({
+                        'url': full_url,
+                        'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
+                        'format_id': format_id,
+                        'quality': 0 if 'HD' in quality else -1,
+                        'preference': 1,
+                    })
 
         self._sort_formats(formats)
 

From 820b06480493ab1b3c146c3edfc2c89fc80061cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 20:48:17 +0600
Subject: [PATCH 0264/1515] [srf] Extract subtitles

---
 youtube_dl/extractor/srf.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 4be329a2f..a080eb7ca 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -83,6 +83,16 @@ class SrfIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        subtitles = {}
+        subtitles_data = video_data.find('Subtitles')
+        if subtitles_data is not None:
+            subtitles_list = [{
+                'url': sub.text,
+                'ext': determine_ext(sub.text),
+            } for sub in subtitles_data]
+            if subtitles_list:
+                subtitles['de'] = subtitles_list
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -90,4 +100,5 @@ class SrfIE(InfoExtractor):
             'title': title,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
+            'subtitles': subtitles,
         }

From fbbb2194097afcc3d3fb4f28888279f1b970b253 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:28:21 +0600
Subject: [PATCH 0265/1515] [srf] Fix direct links ext

---
 youtube_dl/extractor/srf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index a080eb7ca..2ac55dcd1 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -75,7 +75,7 @@ class SrfIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': full_url,
-                        'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
+                        'ext': original_ext,
                         'format_id': format_id,
                         'quality': 0 if 'HD' in quality else -1,
                         'preference': 1,

From 89c09e2a0884301e405d014fdc9706cf1750f798 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:30:13 +0600
Subject: [PATCH 0266/1515] [srf] Update test

---
 youtube_dl/extractor/srf.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 2ac55dcd1..e30351602 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -26,18 +26,15 @@ class SrfIE(InfoExtractor):
     }, {
         # No Speichern (Save) button
         'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
+        'md5': 'd97e236e80d1d24729e5d0953d276a4f',
         'info_dict': {
             'id': '677f5829-e473-4823-ac83-a1087fe97faa',
             'display_id': 'jaguar-xk120-shadow-und-tornado-dampflokomotive',
-            'ext': 'mp4',
+            'ext': 'flv',
             'upload_date': '20130710',
             'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
             'timestamp': 1373493600,
         },
-        'params': {
-            # Require ffmpeg/avconv
-            'skip_download': True,
-        }
     }, {
         'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'only_matching': True,

From 5cb91ceaa50ffe5b8551075765a8ad920efd0b59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:33:01 +0600
Subject: [PATCH 0267/1515] [pladform] Update test

---
 youtube_dl/extractor/pladform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index abde34b94..551c8c9f0 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -30,7 +30,7 @@ class PladformIE(InfoExtractor):
         'info_dict': {
             'id': '100183293',
             'ext': 'mp4',
-            'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+            'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
             'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 694,

From 62259846816568917d3cff41dbbce24a706c9fc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:37:15 +0600
Subject: [PATCH 0268/1515] [generic] Update pladform embed test

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8c859f068..e645d1bb3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -615,7 +615,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': '100183293',
                 'ext': 'mp4',
-                'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+                'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 694,

From 5141249c5930943cbf6890db0e36ba50f06a20ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:47:42 +0600
Subject: [PATCH 0269/1515] [srf] Extend _VALID_URL

---
 youtube_dl/extractor/srf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index e30351602..25b589cb9 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SrfIE(InfoExtractor):
-    _VALID_URL = r'http://www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=(?P<id>[0-9a-f\-]{36})'
+    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'md5': '4cd93523723beff51bb4bee974ee238d',
@@ -38,6 +38,9 @@ class SrfIE(InfoExtractor):
     }, {
         'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'only_matching': True,
+    }, {
+        'url': 'https://tp.srgssr.ch/p/flash?urn=urn:srf:ais:video:28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 65c1a750f5e5327a2ec8f702509eba05364ca4fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:48:22 +0600
Subject: [PATCH 0270/1515] [srf] Show display_id when present

---
 youtube_dl/extractor/srf.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 25b589cb9..77eec0bc7 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -45,12 +45,12 @@ class SrfIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        display_id = re.match(self._VALID_URL, url).group('display_id') or video_id
 
         video_data = self._download_xml(
             'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
-            video_id)
+            display_id)
 
-        display_id = re.match(self._VALID_URL, url).group('display_id')
         title = xpath_text(
             video_data, './AssetMetadatas/AssetMetadata/title', fatal=True)
         thumbnails = [{
@@ -68,10 +68,10 @@ class SrfIE(InfoExtractor):
                 format_id = '%s-%s' % (quality, item.attrib['protocol'])
                 if original_ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        full_url + '?hdcore=3.4.0', video_id, f4m_id=format_id))
+                        full_url + '?hdcore=3.4.0', display_id, f4m_id=format_id))
                 elif original_ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        full_url, video_id, 'mp4', m3u8_id=format_id))
+                        full_url, display_id, 'mp4', m3u8_id=format_id))
                 else:
                     formats.append({
                         'url': full_url,

From be531ef1ec90cb5ee5b3f02d817a140313b76412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 22:12:38 +0600
Subject: [PATCH 0271/1515] [utils] Fix splitunc deprecation warning

---
 youtube_dl/utils.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52f0dd09a..e628fac81 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -312,17 +312,17 @@ def sanitize_path(s):
     """Sanitizes and normalizes path on Windows"""
     if sys.platform != 'win32':
         return s
-    drive, _ = os.path.splitdrive(s)
-    unc, _ = os.path.splitunc(s)
-    unc_or_drive = unc or drive
-    norm_path = os.path.normpath(remove_start(s, unc_or_drive)).split(os.path.sep)
-    if unc_or_drive:
+    drive_or_unc, _ = os.path.splitdrive(s)
+    if sys.version_info < (2, 7) and not drive_or_unc:
+        drive_or_unc, _ = os.path.splitunc(s)
+    norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
+    if drive_or_unc:
         norm_path.pop(0)
     sanitized_path = [
         path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
         for path_part in norm_path]
-    if unc_or_drive:
-        sanitized_path.insert(0, unc_or_drive + os.path.sep)
+    if drive_or_unc:
+        sanitized_path.insert(0, drive_or_unc + os.path.sep)
     return os.path.join(*sanitized_path)
 
 

From 4aec95f3c932ee7042ca4dcae9fcd8c57341bb55 Mon Sep 17 00:00:00 2001
From: FireDart <firedartonline@gmail.com>
Date: Thu, 16 Apr 2015 18:10:53 -0400
Subject: [PATCH 0272/1515] [gfycat] Updated tests.

---
 youtube_dl/extractor/gfycat.py | 58 +++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index d103693cd..6de78c49d 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -8,31 +8,39 @@ from .common import InfoExtractor
 
 class GfycatIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TESTS = [{
-        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
-        'info_dict': {
-            'id':          'DeadlyDecisiveGermanpinscher',
-            'title':       'Ghost in the Shell',
-            'ext':         'mp4',
-            'upload_date': '20140913'
-        }
-    },{
-        'url': 'http://gfycat.com/pleasinghilariouskusimanse',
-        'info_dict': {
-            'id':          'pleasinghilariouskusimanse',
-            'title':       'PleasingHilariousKusimanse',
-            'ext':         'webm',
-            'upload_date': '20150412'
-        }
-    },{
-        'url': 'http://gfycat.com/requiredunkemptbuzzard',
-        'info_dict': {
-            'id':          'requiredunkemptbuzzard',
-            'title':       'Headshot!',
-            'ext':         'gif',
-            'upload_date': '20150130'
-        }
-    }]
+    _TESTS = [
+        {
+            'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
+            'info_dict': {
+                'id':          'DeadlyDecisiveGermanpinscher',
+                'title':       'Ghost in the Shell',
+                'ext':         'mp4',
+                'upload_date': '20140913'
+            }
+        },{
+            'url': 'http://gfycat.com/pleasinghilariouskusimanse',
+            'info_dict': {
+                'id':          'pleasinghilariouskusimanse',
+                'title':       'PleasingHilariousKusimanse',
+                'ext':         'webm',
+                'upload_date': '20150412'
+            },
+            'params': {
+                'format': 'webm',
+            },
+        },{
+            'url': 'http://gfycat.com/requiredunkemptbuzzard',
+            'info_dict': {
+                'id':          'requiredunkemptbuzzard',
+                'title':       'Headshot!',
+                'ext':         'gif',
+                'upload_date': '20150129'
+            },
+            'params': {
+                'format': 'gif',
+            },
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 954352c4c08dab0dd2d9ca20f5a414a307cea96f Mon Sep 17 00:00:00 2001
From: FireDart <firedartonline@gmail.com>
Date: Thu, 16 Apr 2015 18:11:30 -0400
Subject: [PATCH 0273/1515] [gfycat] Fixed preferences.

---
 youtube_dl/extractor/gfycat.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 6de78c49d..5e70ed3f6 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -63,7 +63,7 @@ class GfycatIE(InfoExtractor):
             'height':     json['height'],
             'fps':        json['frameRate'],
             'filesize':   json['mp4Size'],
-            'preference': '-1'
+            'preference': 2
         }, {
             'format_id': 'webm',
             'ext':       'webm',
@@ -72,7 +72,7 @@ class GfycatIE(InfoExtractor):
             'height':     json['height'],
             'fps':        json['frameRate'],
             'filesize':   json['webmSize'],
-            'preference': 0
+            'preference': 1
         }, {
             'format_id':  'gif',
             'ext':        'gif',
@@ -81,7 +81,7 @@ class GfycatIE(InfoExtractor):
             'height':     json['height'],
             'fps':        json['frameRate'],
             'filesize':   json['gifSize'],
-            'preference': 1
+            'preference': 0
         }]
         
         self._sort_formats(formats)

From f11554092b419baa919875432fe6ebc1f22f5307 Mon Sep 17 00:00:00 2001
From: Tjark Saul <tjark.saul@wachplan.net>
Date: Fri, 17 Apr 2015 09:21:54 +0200
Subject: [PATCH 0274/1515] [Lecture2Go] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/lecture2go.py | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/lecture2go.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..3d6e981b2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -249,6 +249,7 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .lecture2go import Lecture2GoIE
 from .letv import (
     LetvIE,
     LetvTvIE,
diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
new file mode 100644
index 000000000..9cf28e31c
--- /dev/null
+++ b/youtube_dl/extractor/lecture2go.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Lecture2GoIE(InfoExtractor):
+    _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
+        'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
+        'info_dict': {
+            'id': '17473',
+            'ext': 'mp4',
+            'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4',
+            'title': '2 - Endliche Automaten und reguläre Sprachen'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<em class="title">(.*?)</em>', webpage, 'title')
+        video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
+        creator = self._html_search_regex(r'<div id="description">(.*)</div>', webpage, 'creator')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'creator': creator
+        }

From 024ebb270663fbe27cfab52b1a8b9a21f227d985 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 10:46:25 +0200
Subject: [PATCH 0275/1515] [soundcloud] Handle 'secret_token' for
 'w.soundcloud.com/player/?url=*' urls (fixes #5453)

---
 youtube_dl/extractor/soundcloud.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 316b2c90f..7efc6aff1 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -221,7 +221,12 @@ class SoundcloudIE(InfoExtractor):
                 info_json_url += "&secret_token=" + token
         elif mobj.group('player'):
             query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-            return self.url_result(query['url'][0])
+            real_url = query['url'][0]
+            # If the token is in the query of the original url we have to
+            # manually add it
+            if 'secret_token' in query:
+                real_url += '?secret_token=' + query['secret_token'][0]
+            return self.url_result(real_url)
         else:
             # extract uploader (which is in the url)
             uploader = mobj.group('uploader')

From 3220c50f9af3a1df239656025aebfe00fec00e45 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 17 Apr 2015 11:14:25 +0200
Subject: [PATCH 0276/1515] release 2015.04.17

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c85a39918..80e86c1b6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -98,6 +98,7 @@
  - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED
  - **Cracked**
  - **Criterion**
+ - **CrooksAndLiars**
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
@@ -359,6 +360,9 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
+ - **QQMusic**
+ - **QQMusicAlbum**
+ - **QQMusicSinger**
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -434,6 +438,7 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
+ - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1095fea2f..3fd0e7e56 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.09'
+__version__ = '2015.04.17'

From 08f2a92c9c23cf460b00a290b5b3819c7972231b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 14:55:24 +0200
Subject: [PATCH 0277/1515] InfoExtractor._search_regex: Suggest updating when
 the regex is not found (suggested in #5442)

Reuse the same message from ExtractorError
---
 youtube_dl/extractor/common.py |  4 ++--
 youtube_dl/utils.py            | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 28f672e42..7757bf950 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -23,6 +23,7 @@ from ..compat import (
 )
 from ..utils import (
     age_restricted,
+    bug_reports_message,
     clean_html,
     compiled_regex_type,
     ExtractorError,
@@ -556,8 +557,7 @@ class InfoExtractor(object):
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
         else:
-            self._downloader.report_warning('unable to extract %s; '
-                                            'please report this issue on http://yt-dl.org/bug' % _name)
+            self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e628fac81..edeee1853 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -452,6 +452,17 @@ def make_HTTPS_handler(params, **kwargs):
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
+def bug_reports_message():
+    if ytdl_is_updateable():
+        update_cmd = 'type  youtube-dl -U  to update'
+    else:
+        update_cmd = 'see  https://yt-dl.org/update  on how to update'
+    msg = '; please report this issue on https://yt-dl.org/bug .'
+    msg += ' Make sure you are using the latest version; %s.' % update_cmd
+    msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
+    return msg
+
+
 class ExtractorError(Exception):
     """Error during info extraction."""
 
@@ -467,13 +478,7 @@ class ExtractorError(Exception):
         if cause:
             msg += ' (caused by %r)' % cause
         if not expected:
-            if ytdl_is_updateable():
-                update_cmd = 'type  youtube-dl -U  to update'
-            else:
-                update_cmd = 'see  https://yt-dl.org/update  on how to update'
-            msg += '; please report this issue on https://yt-dl.org/bug .'
-            msg += ' Make sure you are using the latest version; %s.' % update_cmd
-            msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
+            msg += bug_reports_message()
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb

From d8e7ef04dcb583f3271a6b6a099a3da2e650fb45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Apr 2015 22:56:26 +0600
Subject: [PATCH 0278/1515] [vimple] Fix extraction (Closes #5448)

---
 youtube_dl/extractor/vimple.py | 71 ++++++++++++----------------------
 1 file changed, 25 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index ee3d86117..aa3d6ddfd 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -1,75 +1,54 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import base64
-import re
-import xml.etree.ElementTree
-import zlib
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
 
 class VimpleIE(InfoExtractor):
-    IE_DESC = 'Vimple.ru'
-    _VALID_URL = r'https?://(player.vimple.ru/iframe|vimple.ru)/(?P<id>[a-f0-9]{10,})'
+    IE_DESC = 'Vimple - one-click video hosting'
+    _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
     _TESTS = [
         {
             'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
             'md5': '2e750a330ed211d3fd41821c6ad9a279',
             'info_dict': {
-                'id': 'c0f6b1687dcd4000a97ebe70068039cf',
+                'id': 'c0f6b168-7dcd-4000-a97e-be70068039cf',
                 'ext': 'mp4',
                 'title': 'Sunset',
                 'duration': 20,
                 'thumbnail': 're:https?://.*?\.jpg',
             },
-        },
+        }, {
+            'url': 'http://player.vimple.ru/iframe/52e1beec-1314-4a83-aeac-c61562eadbf9',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        iframe_url = 'http://player.vimple.ru/iframe/%s' % video_id
+        webpage = self._download_webpage(
+            'http://player.vimple.ru/iframe/%s' % video_id, video_id)
 
-        iframe = self._download_webpage(
-            iframe_url, video_id,
-            note='Downloading iframe', errnote='unable to fetch iframe')
-        player_url = self._html_search_regex(
-            r'"(http://player.vimple.ru/flash/.+?)"', iframe, 'player url')
+        playlist = self._parse_json(
+            self._search_regex(
+                r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
+            video_id)['playlist'][0]
 
-        player = self._request_webpage(
-            player_url, video_id, note='Downloading swf player').read()
+        title = playlist['title']
+        video_id = playlist.get('videoId') or video_id
+        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
+        duration = int_or_none(playlist.get('duration'))
 
-        player = zlib.decompress(player[8:])
-
-        xml_pieces = re.findall(b'([a-zA-Z0-9 =+/]{500})', player)
-        xml_pieces = [piece[1:-1] for piece in xml_pieces]
-
-        xml_data = b''.join(xml_pieces)
-        xml_data = base64.b64decode(xml_data)
-
-        xml_data = xml.etree.ElementTree.fromstring(xml_data)
-
-        video = xml_data.find('Video')
-        quality = video.get('quality')
-        q_tag = video.find(quality.capitalize())
-
-        formats = [
-            {
-                'url': q_tag.get('url'),
-                'tbr': int(q_tag.get('bitrate')),
-                'filesize': int(q_tag.get('filesize')),
-                'format_id': quality,
-            },
-        ]
+        formats = [{
+            'url': f['url'],
+        } for f in playlist['video']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video.find('Title').text,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'formats': formats,
-            'thumbnail': video.find('Poster').get('url'),
-            'duration': int_or_none(video.get('duration')),
-            'webpage_url': video.find('Share').get('videoPageUrl'),
         }

From c5826a491b7b214a7e81030ad53103c4aca04dc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 19:02:49 +0200
Subject: [PATCH 0279/1515] [mixcloud] Simplify url extraction

On the tracks I tested the server number in the url from the webpage is valid
for the mp3 or the m4a file and any other number is invalid, it's a
waste of time to check them.
---
 youtube_dl/extractor/mixcloud.py | 72 +++++++-------------------------
 1 file changed, 15 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 84f291558..425a4ccf1 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -46,20 +45,16 @@ class MixcloudIE(InfoExtractor):
         },
     }]
 
-    def _get_url(self, track_id, template_url, server_number):
-        boundaries = (1, 30)
-        for nr in server_numbers(server_number, boundaries):
-            url = template_url % nr
-            try:
-                # We only want to know if the request succeed
-                # don't download the whole file
-                self._request_webpage(
-                    HEADRequest(url), track_id,
-                    'Checking URL %d/%d ...' % (nr, boundaries[-1]))
-                return url
-            except ExtractorError:
-                pass
-        return None
+    def _check_url(self, url, track_id, ext):
+        try:
+            # We only want to know if the request succeed
+            # don't download the whole file
+            self._request_webpage(
+                HEADRequest(url), track_id,
+                'Trying %s URL' % ext)
+            return True
+        except ExtractorError:
+            return False
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -72,15 +67,10 @@ class MixcloudIE(InfoExtractor):
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
-        server_number = int(self._search_regex(r'stream(\d+)', song_url, 'server number'))
-        template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
-        final_song_url = self._get_url(track_id, template_url, server_number)
-        if final_song_url is None:
-            self.to_screen('Trying with m4a extension')
-            template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            final_song_url = self._get_url(track_id, template_url, server_number)
-        if final_song_url is None:
-            raise ExtractorError('Unable to extract track url')
+        if not self._check_url(song_url, track_id, 'mp3'):
+            song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
+            if not self._check_url(song_url, track_id, 'm4a'):
+                raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
             r'm-play-on-spacebar[^>]+'
@@ -107,7 +97,7 @@ class MixcloudIE(InfoExtractor):
         return {
             'id': track_id,
             'title': title,
-            'url': final_song_url,
+            'url': song_url,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
@@ -115,35 +105,3 @@ class MixcloudIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
         }
-
-
-def server_numbers(first, boundaries):
-    """ Server numbers to try in descending order of probable availability.
-    Starting from first (i.e. the number of the server hosting the preview file)
-    and going further and further up to the higher boundary and down to the
-    lower one in an alternating fashion. Namely:
-
-        server_numbers(2, (1, 5))
-
-        # Where the preview server is 2, min number is 1 and max is 5.
-        # Yields: 2, 3, 1, 4, 5
-
-    Why not random numbers or increasing sequences? Since from what I've seen,
-    full length files seem to be hosted on servers whose number is closer to
-    that of the preview; to be confirmed.
-    """
-    zip_longest = getattr(itertools, 'zip_longest', None)
-    if zip_longest is None:
-        # python 2.x
-        zip_longest = itertools.izip_longest
-
-    if len(boundaries) != 2:
-        raise ValueError("boundaries should be a two-element tuple")
-    min, max = boundaries
-    highs = range(first + 1, max + 1)
-    lows = range(first - 1, min - 1, -1)
-    rest = filter(
-        None, itertools.chain.from_iterable(zip_longest(highs, lows)))
-    yield first
-    for n in rest:
-        yield n

From 214e74bf6f6eee15cf85c9d5a4defce14b56d586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 19:24:30 +0200
Subject: [PATCH 0280/1515] [soundcloud] Raise an error instead of calling
 'report_error'

---
 youtube_dl/extractor/soundcloud.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 7efc6aff1..183ff50f4 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -279,9 +279,8 @@ class SoundcloudSetIE(SoundcloudIE):
         info = self._download_json(resolv_url, full_title)
 
         if 'errors' in info:
-            for err in info['errors']:
-                self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
-            return
+            msgs = (compat_str(err['error_message']) for err in info['errors'])
+            raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
         return {
             '_type': 'playlist',

From 7691a7a3bd77dd2c169a9a8592283d99a9266973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Apr 2015 23:41:07 +0600
Subject: [PATCH 0281/1515] [comedycentral] Fix feed uri request (Closes #5449,
 closes #5455)

---
 youtube_dl/extractor/comedycentral.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e5edcc84b..91ebb0ce5 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -201,7 +201,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
         uri = mMovieParams[0][1]
         # Correct cc.com in uri
-        uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.cc.com', uri)
+        uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.com', uri)
 
         index_url = 'http://%s.cc.com/feeds/mrss?%s' % (show_name, compat_urllib_parse.urlencode({'uri': uri}))
         idoc = self._download_xml(

From ce81b1411d182fbfe7bd6da7b875d50f37ae38d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 22:29:30 +0200
Subject: [PATCH 0282/1515] FFmpegExtractAudioPP: Simplify handling of already
 existing files

---
 youtube_dl/postprocessor/ffmpeg.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 8e99a3c2c..4c4a038f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -264,15 +264,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         new_path = prefix + sep + extension
 
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
-        if new_path == path:
-            self._nopostoverwrites = True
+        if (new_path == path or
+                (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
+            self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
+            return True, information
 
         try:
-            if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
-                self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
-            else:
-                self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
-                self.run_ffmpeg(path, new_path, acodec, more_opts)
+            self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
+            self.run_ffmpeg(path, new_path, acodec, more_opts)
         except AudioConversionError as e:
             raise PostProcessingError(
                 'audio conversion failed: ' + e.msg)
@@ -286,7 +285,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
-        return self._nopostoverwrites, information
+        return False, information
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):

From ecc6bd1341cd03fd335058371c3efa63dfd31152 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 22:38:14 +0200
Subject: [PATCH 0283/1515] YoutubeDL.post_process: simplify keep_video
 handling

Since keep_video started as None we always set it to keep_video_wish unless it was None, so in the end keep_video == keep_video_wish. This should have been changed in f3ff1a3696c4080468e2cc5810c34273b148bd3e, but I didn't notice it.
---
 youtube_dl/YoutubeDL.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a68b24ab4..6ac85f4e7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1486,16 +1486,9 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
-            keep_video = None
             old_filename = info['filepath']
             try:
-                keep_video_wish, info = pp.run(info)
-                if keep_video_wish is not None:
-                    if keep_video_wish:
-                        keep_video = keep_video_wish
-                    elif keep_video is None:
-                        # No clear decision yet, let IE decide
-                        keep_video = keep_video_wish
+                keep_video, info = pp.run(info)
             except PostProcessingError as e:
                 self.report_error(e.msg)
             if keep_video is False and not self.params.get('keepvideo', False):

From f52e66505a9f9403d2278d22732ed98b711292fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:50:22 +0600
Subject: [PATCH 0284/1515] [gfycat] Simplify (Closes #5439, Closes #5394)

---
 youtube_dl/extractor/gfycat.py | 179 ++++++++++++++-------------------
 1 file changed, 77 insertions(+), 102 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 5e70ed3f6..397f1d42e 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -1,115 +1,90 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-import datetime
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    qualities,
+)
+
 
 class GfycatIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TESTS = [
-        {
-            'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
-            'info_dict': {
-                'id':          'DeadlyDecisiveGermanpinscher',
-                'title':       'Ghost in the Shell',
-                'ext':         'mp4',
-                'upload_date': '20140913'
-            }
-        },{
-            'url': 'http://gfycat.com/pleasinghilariouskusimanse',
-            'info_dict': {
-                'id':          'pleasinghilariouskusimanse',
-                'title':       'PleasingHilariousKusimanse',
-                'ext':         'webm',
-                'upload_date': '20150412'
-            },
-            'params': {
-                'format': 'webm',
-            },
-        },{
-            'url': 'http://gfycat.com/requiredunkemptbuzzard',
-            'info_dict': {
-                'id':          'requiredunkemptbuzzard',
-                'title':       'Headshot!',
-                'ext':         'gif',
-                'upload_date': '20150129'
-            },
-            'params': {
-                'format': 'gif',
-            },
-        },
-    ]
+    _TEST = {
+        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
+        'info_dict': {
+            'id': 'DeadlyDecisiveGermanpinscher',
+            'ext': 'mp4',
+            'title': 'Ghost in the Shell',
+            'timestamp': 1410656006,
+            'upload_date': '20140914',
+            'uploader': 'anonymous',
+            'duration': 10.4,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'categories': list,
+            'age_limit': 0,
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json     = self._download_json("http://gfycat.com/cajax/get/" + video_id, video_id, 'Downloading video info')['gfyItem']
-        
-        # Title
-        # Use user title first, else fallback to url formated name
-        if json['title']:
-            video_title = json['title']
-        else:
-            video_title = json['gfyName']
-        
-        # Formats
-        # Pref: mp4, webm, gif
-        formats = [{
-            'format_id':  'mp4',
-            'ext':        'mp4',
-            'url':        json['mp4Url'],
-            'width':      json['width'],
-            'height':     json['height'],
-            'fps':        json['frameRate'],
-            'filesize':   json['mp4Size'],
-            'preference': 2
-        }, {
-            'format_id': 'webm',
-            'ext':       'webm',
-            'url':        json['webmUrl'],
-            'width':      json['width'],
-            'height':     json['height'],
-            'fps':        json['frameRate'],
-            'filesize':   json['webmSize'],
-            'preference': 1
-        }, {
-            'format_id':  'gif',
-            'ext':        'gif',
-            'url':        json['gifUrl'],
-            'width':      json['width'],
-            'height':     json['height'],
-            'fps':        json['frameRate'],
-            'filesize':   json['gifSize'],
-            'preference': 0
-        }]
-        
+
+        gfy = self._download_json(
+            'http://gfycat.com/cajax/get/%s' % video_id,
+            video_id, 'Downloading video info')['gfyItem']
+
+        title = gfy.get('title') or gfy['gfyName']
+        description = gfy.get('description')
+        timestamp = int_or_none(gfy.get('createDate'))
+        uploader = gfy.get('userName')
+        view_count = int_or_none(gfy.get('views'))
+        like_count = int_or_none(gfy.get('likes'))
+        dislike_count = int_or_none(gfy.get('dislikes'))
+        age_limit = 18 if gfy.get('nsfw') == '1' else 0
+
+        width = int_or_none(gfy.get('width'))
+        height = int_or_none(gfy.get('height'))
+        fps = int_or_none(gfy.get('frameRate'))
+        num_frames = int_or_none(gfy.get('numFrames'))
+
+        duration = float_or_none(num_frames, fps) if num_frames and fps else None
+
+        categories = gfy.get('tags') or gfy.get('extraLemmas') or []
+
+        FORMATS = ('gif', 'webm', 'mp4')
+        quality = qualities(FORMATS)
+
+        formats = []
+        for format_id in FORMATS:
+            video_url = gfy.get('%sUrl' % format_id)
+            if not video_url:
+                continue
+            filesize = gfy.get('%sSize' % format_id)
+            formats.append({
+                'url': video_url,
+                'format_id': format_id,
+                'width': width,
+                'height': height,
+                'fps': fps,
+                'filesize': filesize,
+                'quality': quality(format_id),
+            })
         self._sort_formats(formats)
-        
-        # Date
-        date = datetime.datetime.fromtimestamp(
-            int(json['createDate'])
-        ).strftime('%Y%m%d')
-        
-        # Length
-        duration = json['numFrames'] / json['frameRate']
-        
-        # Age limit
-        # 1 = nsfw / 0 = sfw
-        if json['nsfw'] == 1:
-            age_limit = 18
-        else:
-            age_limit = 0
-        
+
         return {
-            'id':          video_id,
-            'title':       video_title,
-            'formats':     formats,
-            'creator':     json['userName'],
-            'description': json['description'],
-            'upload_date': date,
-            'categories':  json['tags'],
-            'age_limit':   age_limit,
-            'duration':    duration,
-            'view_count':  json['views']
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'duration': duration,
+            'view_count': view_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'categories': categories,
+            'age_limit': age_limit,
+            'formats': formats,
         }

From bf12cbe07c3a22deb46848df3bd3242da645eab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:51:21 +0600
Subject: [PATCH 0285/1515] Credit @julianrichen for gfycat (#5440)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index db3f42b26..267b8da1e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -123,3 +123,4 @@ Will W.
 Mohammad Teimori Pabandi
 Roman Le Négrate
 Matthias Küch
+Julian Richen

From c62566971fb8877d5ac52bc613d3bd53fe87a8d8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Apr 2015 00:08:52 +0800
Subject: [PATCH 0286/1515] [facebook] Extend _VALID_URL

---
 youtube_dl/extractor/facebook.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f0e575320..f3a68b21d 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -24,8 +24,12 @@ class FacebookIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://(?:\w+\.)?facebook\.com/
         (?:[^#]*?\#!/)?
-        (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
-        (?:v|video_id)=(?P<id>[0-9]+)
+        (?:
+            (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
+            (?:v|video_id)=|
+            [^/]+/videos/
+        )
+        (?P<id>[0-9]+)
         (?:.*)'''
     _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
@@ -50,6 +54,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
+        'only_matching': True,
     }]
 
     def _login(self):

From 53faa3ca5f62c46bb56c0a85d1ed87b911b7ffa4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Apr 2015 16:08:24 +0800
Subject: [PATCH 0287/1515] [facebook] Extend _VALID_URL take 2 (#5120)

---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f3a68b21d..937b28fcc 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -27,7 +27,7 @@ class FacebookIE(InfoExtractor):
         (?:
             (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
             (?:v|video_id)=|
-            [^/]+/videos/
+            [^/]+/videos/(?:[^/]+/)?
         )
         (?P<id>[0-9]+)
         (?:.*)'''
@@ -57,6 +57,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
         'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
+        'only_matching': True,
     }]
 
     def _login(self):

From 592e97e8550389e22b716eb33c30584aa3a8d656 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 11:36:42 +0200
Subject: [PATCH 0288/1515] Postprocessors: use a list for the files that can
 be deleted

We could only know if we had to delete the original file, but this system allows to specify us more files (like subtitles).
---
 test/test_YoutubeDL.py                        | 21 ++++++++----
 youtube_dl/YoutubeDL.py                       | 13 +++----
 youtube_dl/postprocessor/atomicparsley.py     |  2 +-
 youtube_dl/postprocessor/common.py            |  8 ++---
 youtube_dl/postprocessor/execafterdownload.py |  2 +-
 youtube_dl/postprocessor/ffmpeg.py            | 34 +++++++++----------
 youtube_dl/postprocessor/metadatafromtitle.py |  2 +-
 youtube_dl/postprocessor/xattrpp.py           |  4 +--
 8 files changed, 48 insertions(+), 38 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 652519831..820e55ec2 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -443,27 +443,36 @@ class TestYoutubeDL(unittest.TestCase):
             def run(self, info):
                 with open(audiofile, 'wt') as f:
                     f.write('EXAMPLE')
-                info['filepath']
-                return False, info
+                return [info['filepath']], info
 
-        def run_pp(params):
+        def run_pp(params, PP):
             with open(filename, 'wt') as f:
                 f.write('EXAMPLE')
             ydl = YoutubeDL(params)
-            ydl.add_post_processor(SimplePP())
+            ydl.add_post_processor(PP())
             ydl.post_process(filename, {'filepath': filename})
 
-        run_pp({'keepvideo': True})
+        run_pp({'keepvideo': True}, SimplePP)
         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(filename)
         os.unlink(audiofile)
 
-        run_pp({'keepvideo': False})
+        run_pp({'keepvideo': False}, SimplePP)
         self.assertFalse(os.path.exists(filename), '%s exists' % filename)
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(audiofile)
 
+        class ModifierPP(PostProcessor):
+            def run(self, info):
+                with open(info['filepath'], 'wt') as f:
+                    f.write('MODIFIED')
+                return [], info
+
+        run_pp({'keepvideo': False}, ModifierPP)
+        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
+        os.unlink(filename)
+
     def test_match_filter(self):
         class FilterYDL(YDL):
             def __init__(self, *args, **kwargs):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6ac85f4e7..8d8b146b2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1488,15 +1488,16 @@ class YoutubeDL(object):
         for pp in pps_chain:
             old_filename = info['filepath']
             try:
-                keep_video, info = pp.run(info)
+                files_to_delete, info = pp.run(info)
             except PostProcessingError as e:
                 self.report_error(e.msg)
-            if keep_video is False and not self.params.get('keepvideo', False):
-                try:
+            if files_to_delete and not self.params.get('keepvideo', False):
+                for old_filename in files_to_delete:
                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
-                    os.remove(encodeFilename(old_filename))
-                except (IOError, OSError):
-                    self.report_warning('Unable to remove downloaded video file')
+                    try:
+                        os.remove(encodeFilename(old_filename))
+                    except (IOError, OSError):
+                        self.report_warning('Unable to remove downloaded original file')
 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case
diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
index a5dfc136a..e4e198695 100644
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ b/youtube_dl/postprocessor/atomicparsley.py
@@ -59,4 +59,4 @@ class AtomicParsleyPP(PostProcessor):
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index ef9fdfa19..3b0e8ddd8 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -42,14 +42,14 @@ class PostProcessor(object):
         one has an extra field called "filepath" that points to the
         downloaded file.
 
-        This method returns a tuple, the first element of which describes
-        whether the original file should be kept (i.e. not deleted - None for
-        no preference), and the second of which is the updated information.
+        This method returns a tuple, the first element is a list of the files
+        that can be deleted, and the second of which is the updated
+        information.
 
         In addition, this method may raise a PostProcessingError
         exception if post processing fails.
         """
-        return None, information  # by default, keep file and do nothing
+        return [], information  # by default, keep file and do nothing
 
     def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
         try:
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 75c0f7bbe..341437575 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -25,4 +25,4 @@ class ExecAfterDownloadPP(PostProcessor):
             raise PostProcessingError(
                 'Command returned error code %d' % retCode)
 
-        return None, information  # by default, keep file and do nothing
+        return [], information
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4c4a038f9..4cdbfce63 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -267,7 +267,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         if (new_path == path or
                 (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
             self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
-            return True, information
+            return [], information
 
         try:
             self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
@@ -285,7 +285,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
-        return False, information
+        return [path], information
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
@@ -299,13 +299,13 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
         outpath = prefix + sep + self._preferedformat
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
-            return True, information
+            return [], information
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, [])
         information['filepath'] = outpath
         information['format'] = self._preferedformat
         information['ext'] = self._preferedformat
-        return False, information
+        return [path], information
 
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
@@ -505,11 +505,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
     def run(self, information):
         if information['ext'] != 'mp4':
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
-            return True, information
+            return [], information
         subtitles = information.get('requested_subtitles')
         if not subtitles:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
-            return True, information
+            return [], information
 
         sub_langs = list(subtitles.keys())
         filename = information['filepath']
@@ -535,7 +535,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, information
+        return [], information
 
 
 class FFmpegMetadataPP(FFmpegPostProcessor):
@@ -561,7 +561,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')
-            return True, info
+            return [], info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -578,7 +578,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
-        return True, info
+        return [], info
 
 
 class FFmpegMergerPP(FFmpegPostProcessor):
@@ -587,7 +587,7 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
-        return True, info
+        return [], info
 
 
 class FFmpegAudioFixPP(FFmpegPostProcessor):
@@ -602,14 +602,14 @@ class FFmpegAudioFixPP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
 
 
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')
         if stretched_ratio is None or stretched_ratio == 1:
-            return True, info
+            return [], info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -621,13 +621,13 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
 
 
 class FFmpegFixupM4aPP(FFmpegPostProcessor):
     def run(self, info):
         if info.get('container') != 'm4a_dash':
-            return True, info
+            return [], info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -639,7 +639,7 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
 
 
 class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
@@ -656,7 +656,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             new_format = 'webvtt'
         if subs is None:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
-            return True, info
+            return [], info
         self._downloader.to_screen('[ffmpeg] Converting subtitles')
         for lang, sub in subs.items():
             ext = sub['ext']
@@ -676,4 +676,4 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'data': f.read(),
                 }
 
-        return True, info
+        return [], info
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 5019433d3..a56077f20 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -44,4 +44,4 @@ class MetadataFromTitlePP(PostProcessor):
             info[attribute] = value
             self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)
 
-        return True, info
+        return [], info
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index f6c63fe97..0cba99fc3 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -105,8 +105,8 @@ class XAttrMetadataPP(PostProcessor):
                     byte_value = value.encode('utf-8')
                     write_xattr(filename, xattrname, byte_value)
 
-            return True, info
+            return [], info
 
         except (subprocess.CalledProcessError, OSError):
             self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
-            return False, info
+            return [], info

From 14523ed9695975704fae441a3518daf8b0e382fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 11:44:42 +0200
Subject: [PATCH 0289/1515] FFmpegEmbedSubtitlePP: remove the subtitle files if
 '--keep-video' is not given (closes #5435)

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4cdbfce63..852dc3e44 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -513,7 +513,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 
         sub_langs = list(subtitles.keys())
         filename = information['filepath']
-        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
+        sub_filenames = [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
+        input_files = [filename] + sub_filenames
 
         opts = [
             '-map', '0',
@@ -535,7 +536,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return [], information
+        return sub_filenames, information
 
 
 class FFmpegMetadataPP(FFmpegPostProcessor):

From d47aeb2252aa0289199bc3ea9bb533997051ec0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 11:52:36 +0200
Subject: [PATCH 0290/1515] FFmpegMergerPP: use the new system for specifying
 which files can be delete

---
 youtube_dl/YoutubeDL.py            | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 9 ++-------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8d8b146b2..944571881 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1361,7 +1361,7 @@ class YoutubeDL(object):
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True
-                    merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
+                    merger = FFmpegMergerPP(self)
                     if not merger.available:
                         postprocessors = []
                         self.report_warning('You have requested multiple '
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 852dc3e44..4d619236e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -28,9 +28,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None, deletetempfiles=False):
+    def __init__(self, downloader=None):
         PostProcessor.__init__(self, downloader)
-        self._deletetempfiles = deletetempfiles
         self._determine_executables()
 
     def check_version(self):
@@ -148,10 +147,6 @@ class FFmpegPostProcessor(PostProcessor):
             raise FFmpegPostProcessorError(msg)
         self.try_utime(out_path, oldest_mtime, oldest_mtime)
 
-        if self._deletetempfiles:
-            for ipath in input_paths:
-                os.remove(ipath)
-
     def run_ffmpeg(self, path, out_path, opts):
         self.run_ffmpeg_multiple_files([path], out_path, opts)
 
@@ -588,7 +583,7 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
-        return [], info
+        return info['__files_to_merge'], info
 
 
 class FFmpegAudioFixPP(FFmpegPostProcessor):

From cc36e2295a19796380e4a4f2abf618e727617efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 13:27:35 +0200
Subject: [PATCH 0291/1515] [ign] Fix extraction of some videos in articles

Give higher preference to the hero-poster regex because some articles may contain other videos
---
 youtube_dl/extractor/ign.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 3aade9e74..bf2d2041b 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -61,7 +61,7 @@ class IGNIE(InfoExtractor):
         },
         {
             'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
-            'md5': '4e9a0bda1e5eebd31ddcf86ec0b9b3c7',
+            'md5': '618fedb9c901fd086f6f093564ef8558',
             'info_dict': {
                 'id': '078fdd005f6d3c02f63d795faa1b984f',
                 'ext': 'mp4',
@@ -77,10 +77,10 @@ class IGNIE(InfoExtractor):
     def _find_video_id(self, webpage):
         res_id = [
             r'"video_id"\s*:\s*"(.*?)"',
+            r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
             r'data-video-id="(.+?)"',
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
-            r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
         ]
         return self._search_regex(res_id, webpage, 'video id')
 

From 8f4e8bf28080a7d6e969fea66e59afd3a1b2f085 Mon Sep 17 00:00:00 2001
From: hedii <hedi.chaibs@gmail.com>
Date: Sat, 18 Apr 2015 15:40:40 +0200
Subject: [PATCH 0292/1515] Update wat.py

line 116, modify 'Downloding' to 'Downloading'.
It looks like nothing, but it is very annoying when youtube-dl command's output is parsed to find progress on a php (or other language) website for example.
---
 youtube_dl/extractor/wat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index bf9e40bad..affcc52f6 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -113,7 +113,7 @@ class WatIE(InfoExtractor):
             video_url = self._download_webpage(
                 'http://www.wat.tv/get%s?token=%s&getURL=1&country=%s' % (webid, compute_token(webid), country),
                 real_id,
-                'Downloding %s video URL' % fmt[0],
+                'Downloading %s video URL' % fmt[0],
                 'Failed to download %s video URL' % fmt[0],
                 False)
             if not video_url:

From fec2d97ca2f1aa8b64c24b28d8c63cab052e9db4 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Thu, 15 Jan 2015 21:28:57 -0500
Subject: [PATCH 0293/1515] Add megavideoz.eu support.

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/megavideozeu.py | 39 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9e9e20589..6657695cb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
+from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
new file mode 100644
index 000000000..e77b5f734
--- /dev/null
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+)
+
+
+class MegavideozeuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        tmp_video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, tmp_video_id)
+
+        config_php = self._html_search_regex(
+            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+
+	configpage = self._download_webpage(config_php, tmp_video_id)
+
+        video_id = self._html_search_regex(
+            r'<mediaid>([^<]+)', configpage, 'video id')
+        video_url = self._html_search_regex(
+            r'<file>([^<]+)', configpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
+        duration = int_or_none(self._html_search_regex(
+            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'duration': duration
+        }

From f32cb5cb147e8e6f05625ed6b5880eb566101f03 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Fri, 17 Apr 2015 11:25:01 -0400
Subject: [PATCH 0294/1515] [megavideoez] Add working test

---
 youtube_dl/extractor/megavideozeu.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
index e77b5f734..ee26b0f2e 100644
--- a/youtube_dl/extractor/megavideozeu.py
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -11,6 +11,18 @@ from ..utils import (
 
 class MegavideozeuIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+    _TESTS = [
+        {
+            'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
+            'info_dict': {
+                'id': '48723',
+                'ext': 'mp4',
+                'duration': '10',
+                'title': 'SMPTE Universal Film Leader',
+            }
+        }
+    ]
+
 
     def _real_extract(self, url):
         tmp_video_id = self._match_id(url)
@@ -29,7 +41,7 @@ class MegavideozeuIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
         duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+            r'<duration>([0-9\.]+)', configpage, 'duration', fatal=False))
 
         return {
             'id': video_id,

From 31f224008e39b629d793a8fc3f8fafc7a4baf417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:07:45 +0600
Subject: [PATCH 0295/1515] [megavideozeu] Simplify (Closes #5454)

---
 youtube_dl/extractor/megavideozeu.py | 59 ++++++++++++++--------------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
index ee26b0f2e..f98080caa 100644
--- a/youtube_dl/extractor/megavideozeu.py
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -1,51 +1,52 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
+    float_or_none,
+    xpath_text,
 )
 
 
 class MegavideozeuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
-    _TESTS = [
-        {
-            'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
-            'info_dict': {
-                'id': '48723',
-                'ext': 'mp4',
-                'duration': '10',
-                'title': 'SMPTE Universal Film Leader',
-            }
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
+    _TEST = {
+        'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
+        'info_dict': {
+            'id': '48723',
+            'display_id': 'SMPTE-Universal-Film-Leader',
+            'ext': 'mp4',
+            'title': 'SMPTE Universal Film Leader',
+            'thumbnail': 're:https?://.*?\.jpg',
+            'duration': 10.93,
         }
-    ]
-
+    }
 
     def _real_extract(self, url):
-        tmp_video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
 
-        webpage = self._download_webpage(url, tmp_video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        config_php = self._html_search_regex(
-            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+        config = self._download_xml(
+            self._search_regex(
+                r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),
+            display_id)
 
-	configpage = self._download_webpage(config_php, tmp_video_id)
-
-        video_id = self._html_search_regex(
-            r'<mediaid>([^<]+)', configpage, 'video id')
-        video_url = self._html_search_regex(
-            r'<file>([^<]+)', configpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
-        duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9\.]+)', configpage, 'duration', fatal=False))
+        video_url = xpath_text(config, './file', 'video url', fatal=True)
+        title = xpath_text(config, './title', 'title', fatal=True)
+        thumbnail = xpath_text(config, './image', 'thumbnail')
+        duration = float_or_none(xpath_text(config, './duration', 'duration'))
+        video_id = xpath_text(config, './mediaid', 'video id') or video_id
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
+            'thumbnail': thumbnail,
             'duration': duration
         }

From cc9b9df0b680da1cf610da57fbf397d8ebd75e74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:08:29 +0600
Subject: [PATCH 0296/1515] [megavideozeu] Rename extractor

---
 youtube_dl/extractor/__init__.py                        | 2 +-
 youtube_dl/extractor/{megavideozeu.py => megavideoz.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{megavideozeu.py => megavideoz.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6657695cb..1dabea7d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideozeu import MegavideozeuIE
+from .megavideoz import MegaVideozIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideoz.py
similarity index 97%
rename from youtube_dl/extractor/megavideozeu.py
rename to youtube_dl/extractor/megavideoz.py
index f98080caa..d80f3633e 100644
--- a/youtube_dl/extractor/megavideozeu.py
+++ b/youtube_dl/extractor/megavideoz.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class MegavideozeuIE(InfoExtractor):
+class MegaVideozIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
     _TEST = {
         'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',

From 6e218b3f9a57f42756aa3016fa9dd530fb58c452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:09:01 +0600
Subject: [PATCH 0297/1515] [megavideoz] Check non-existing videos

---
 youtube_dl/extractor/megavideoz.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
index d80f3633e..a15b6a32a 100644
--- a/youtube_dl/extractor/megavideoz.py
+++ b/youtube_dl/extractor/megavideoz.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     float_or_none,
     xpath_text,
 )
@@ -31,6 +32,9 @@ class MegaVideozIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        if '>Video Not Found<' in webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         config = self._download_xml(
             self._search_regex(
                 r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),

From 163965d86188e5545caae4b5e33ff25278fee5e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:14:58 +0600
Subject: [PATCH 0298/1515] [megavideoz] Improve non-existing videos check

---
 youtube_dl/extractor/megavideoz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
index a15b6a32a..af7ff07ea 100644
--- a/youtube_dl/extractor/megavideoz.py
+++ b/youtube_dl/extractor/megavideoz.py
@@ -32,7 +32,7 @@ class MegaVideozIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        if '>Video Not Found<' in webpage:
+        if any(p in webpage for p in ('>Video Not Found<', '>404 Error<')):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         config = self._download_xml(

From 880ee801cf8a67643cd0a1130ea46cf3be046bf1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Apr 2015 19:08:37 +0800
Subject: [PATCH 0299/1515] [tests] Allow multi_video to be tested as playlists

---
 test/helper.py        | 2 +-
 test/test_download.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 12afdf184..e1129e58f 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -150,7 +150,7 @@ def expect_info_dict(self, got_dict, expected_dict):
                              'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
     # Check for the presence of mandatory fields
-    if got_dict.get('_type') != 'playlist':
+    if got_dict.get('_type') not in ('playlist', 'multi_video'):
         for key in ('id', 'url', 'title', 'ext'):
             self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
     # Check for mandatory fields that are automatically set by YoutubeDL
diff --git a/test/test_download.py b/test/test_download.py
index 6a149ae4f..1110357a7 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -153,7 +153,7 @@ def generator(test_case):
                     break
 
             if is_playlist:
-                self.assertEqual(res_dict['_type'], 'playlist')
+                self.assertTrue(res_dict['_type'] in ['playlist', 'multi_video'])
                 self.assertTrue('entries' in res_dict)
                 expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
 

From 8b0e8990c241789d7deb1f5e27fc1aea00b4fa5e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Apr 2015 19:12:23 +0800
Subject: [PATCH 0300/1515] [miomio] Replace the slow test case

MioMio_1 takes about 25~35 seconds on information retrieval
---
 youtube_dl/extractor/miomio.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index d41195a96..a784fc5fb 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -31,6 +31,14 @@ class MioMioIE(InfoExtractor):
             'title': '《动漫同人插画绘制》',
         },
         'playlist_mincount': 86,
+        'skip': 'This video takes time too long for retrieving the URL',
+    }, {
+        'url': 'http://www.miomio.tv/watch/cc173113/',
+        'info_dict': {
+            'id': '173113',
+            'title': 'The New Macbook 2015 上手试玩与简评'
+        },
+        'playlist_mincount': 2,
     }]
 
     def _real_extract(self, url):

From f158799bbe72e1fe99ec057cc968d3ec874fb1dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Apr 2015 19:19:44 +0800
Subject: [PATCH 0301/1515] [Sohu] Fix title extraction

---
 youtube_dl/extractor/sohu.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 11edf616a..f8a4840f7 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -47,6 +47,7 @@ class SohuIE(InfoExtractor):
         'url': 'http://my.tv.sohu.com/pl/8384802/78910339.shtml',
         'info_dict': {
             'id': '78910339',
+            'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
             'md5': 'bdbfb8f39924725e6589c146bc1883ad',
@@ -110,7 +111,7 @@ class SohuIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        title = re.sub(r' - 搜狐视频$', '', self._og_search_title(webpage))
 
         vid = self._html_search_regex(
             r'var vid ?= ?["\'](\d+)["\']',
@@ -172,9 +173,10 @@ class SohuIE(InfoExtractor):
             info['id'] = video_id
         else:
             info = {
-                '_type': 'playlist',
+                '_type': 'multi_video',
                 'entries': playlist,
                 'id': video_id,
+                'title': title,
             }
 
         return info

From 5b5fbc0867f0eb73416c10c9d692fceee92b5766 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Apr 2015 16:56:22 +0200
Subject: [PATCH 0302/1515] Detect already merged videos

Without the '--keep-video' option the two files would be downloaded again and even using the option, ffmpeg would be run again, which for some videos can take a long time.
We use a temporary file with ffmpeg so that the final file only exists if it success
---
 youtube_dl/YoutubeDL.py            | 25 +++++++++++++++----------
 youtube_dl/postprocessor/ffmpeg.py |  4 +++-
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 944571881..28cf1662e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1369,16 +1369,21 @@ class YoutubeDL(object):
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]
-                    for f in info_dict['requested_formats']:
-                        new_info = dict(info_dict)
-                        new_info.update(f)
-                        fname = self.prepare_filename(new_info)
-                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                        downloaded.append(fname)
-                        partial_success = dl(fname, new_info)
-                        success = success and partial_success
-                    info_dict['__postprocessors'] = postprocessors
-                    info_dict['__files_to_merge'] = downloaded
+                    if os.path.exists(encodeFilename(filename)):
+                        self.to_screen(
+                            '[download] %s has already been downloaded and '
+                            'merged' % filename)
+                    else:
+                        for f in info_dict['requested_formats']:
+                            new_info = dict(info_dict)
+                            new_info.update(f)
+                            fname = self.prepare_filename(new_info)
+                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                            downloaded.append(fname)
+                            partial_success = dl(fname, new_info)
+                            success = success and partial_success
+                        info_dict['__postprocessors'] = postprocessors
+                        info_dict['__files_to_merge'] = downloaded
                 else:
                     # Just a single file
                     success = dl(filename, info_dict)
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4d619236e..df6fb6665 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -580,9 +580,11 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
-        self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
+        self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args)
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return info['__files_to_merge'], info
 
 

From feccf29c876869f44a9c983977371073b9801a51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 01:14:10 +0600
Subject: [PATCH 0303/1515] [YoutubeDL] Make `bestvideo+bestaudio/best` default
 format when merger is available

---
 youtube_dl/YoutubeDL.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 28cf1662e..e5d497b3f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1091,7 +1091,11 @@ class YoutubeDL(object):
 
         req_format = self.params.get('format')
         if req_format is None:
-            req_format = 'best'
+            req_format_list = []
+            if info_dict['extractor'] == 'youtube' and FFmpegMergerPP(self).available:
+                req_format_list.append('bestvideo+bestaudio')
+            req_format_list.append('best')
+            req_format = '/'.join(req_format_list)
         formats_to_download = []
         if req_format == 'all':
             formats_to_download = formats

From 81cd954a512e48fb5cccc3159cd0581088bff0b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:00:35 +0600
Subject: [PATCH 0304/1515] [YoutubeDL] Merge incompatible formats into mkv
 (#5456)

---
 youtube_dl/YoutubeDL.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5d497b3f..5dd9d2430 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1373,12 +1373,34 @@ class YoutubeDL(object):
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]
+
+                    def compatible_formats(formats):
+                        video, audio = formats
+                        # Check extension
+                        video_ext, audio_ext = audio.get('ext'), video.get('ext')
+                        if video_ext and audio_ext:
+                            COMPATIBLE_EXTS = (
+                                ('mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
+                                ('webm')
+                            )
+                            for exts in COMPATIBLE_EXTS:
+                                if video_ext in exts and audio_ext in exts:
+                                    return True
+                        # TODO: Check acodec/vcodec
+                        return False
+
+                    requested_formats = info_dict['requested_formats']
+                    # Merge incompatible formats into mkv
+                    if not compatible_formats(requested_formats):
+                        filename = os.path.splitext(filename)[0] + '.mkv'
+                        self.report_warning('You have requested formats uncompatible for merge. '
+                                            'The formats will be merged into mkv')
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(
                             '[download] %s has already been downloaded and '
                             'merged' % filename)
                     else:
-                        for f in info_dict['requested_formats']:
+                        for f in requested_formats:
                             new_info = dict(info_dict)
                             new_info.update(f)
                             fname = self.prepare_filename(new_info)

From 70947ea7b13d22a55756dc2b6d086e058edd91b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:07:59 +0600
Subject: [PATCH 0305/1515] [parameters.json] Set default `format` parameter to
 `best`

---
 test/parameters.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/parameters.json b/test/parameters.json
index cbff9bd16..48b5a062e 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -7,7 +7,7 @@
     "forcethumbnail": false, 
     "forcetitle": false, 
     "forceurl": false, 
-    "format": null, 
+    "format": "best",
     "format_limit": null, 
     "ignoreerrors": false, 
     "listformats": null, 

From c0dea0a782a1035225ad25d556e398e2909f62db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 22:33:52 +0600
Subject: [PATCH 0306/1515] [YoutubeDL] Respect explicit
 `--merge-format-output` for uncompatible formats as well

---
 youtube_dl/YoutubeDL.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5dd9d2430..3bb350e2a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1390,8 +1390,7 @@ class YoutubeDL(object):
                         return False
 
                     requested_formats = info_dict['requested_formats']
-                    # Merge incompatible formats into mkv
-                    if not compatible_formats(requested_formats):
+                    if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         filename = os.path.splitext(filename)[0] + '.mkv'
                         self.report_warning('You have requested formats uncompatible for merge. '
                                             'The formats will be merged into mkv')

From a38050925958c48175bf56a988e9714edef8f12f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Apr 2015 18:37:38 +0200
Subject: [PATCH 0307/1515] Move the documentation for the `--format` option to
 the manpage

It's too big for beeing embedded in the help message and it's easier to edit in the markdown file.
---
 README.md             | 19 +++++++++----------
 youtube_dl/options.py | 27 +--------------------------
 2 files changed, 10 insertions(+), 36 deletions(-)

diff --git a/README.md b/README.md
index caa1478d9..68426b685 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,7 @@ youtube-dl - download videos from youtube.com or other video platforms
 - [OPTIONS](#options)
 - [CONFIGURATION](#configuration)
 - [OUTPUT TEMPLATE](#output-template)
+- [FORMAT SELECTION](#format-selection)
 - [VIDEO SELECTION](#video-selection)
 - [FAQ](#faq)
 - [DEVELOPER INSTRUCTIONS](#developer-instructions)
@@ -184,16 +185,7 @@ which means you can modify it, redistribute it or use it however you like.
     --sleep-interval SECONDS         Number of seconds to sleep before each download.
 
 ## Video Format Options:
-    -f, --format FORMAT              Video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
-                                     extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio",
-                                     "worst".  You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]").
-                                     This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec,
-                                     vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a
-                                     question mark (?) after the operator. You can combine format filters, so  -f "[height <=? 720][tbr>500]" selects up to 720p videos
-                                     (or videos where the height is not known) with a bitrate of at least 500 KBit/s. By default, youtube-dl will pick the best quality.
-                                     Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio
-                                     of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f
-                                     bestvideo+bestaudio.
+    -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
     --max-quality FORMAT             Highest quality format to download
@@ -271,6 +263,13 @@ $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filena
 youtube-dl_test_video_.mp4          # A simple file name
 ```
 
+# FORMAT SELECTION
+
+By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
+The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
+
+If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+
 # VIDEO SELECTION
 
 Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 11603f60d..39c38c980 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -322,32 +322,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
-        help=(
-            'Video format code, specify the order of preference using'
-            ' slashes, as in -f 22/17/18 . '
-            ' Instead of format codes, you can select by extension for the '
-            'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
-            'You can also use the special names "best",'
-            ' "bestvideo", "bestaudio", "worst". '
-            ' You can filter the video results by putting a condition in'
-            ' brackets, as in -f "best[height=720]"'
-            ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, vbr, asr, and fps'
-            ' and the comparisons <, <=, >, >=, =, !='
-            ' and for ext, acodec, vcodec, container, and protocol'
-            ' and the comparisons =, != .'
-            ' Formats for which the value is not known are excluded unless you'
-            ' put a question mark (?) after the operator.'
-            ' You can combine format filters, so  '
-            '-f "[height <=? 720][tbr>500]" '
-            'selects up to 720p videos (or videos where the height is not '
-            'known) with a bitrate of at least 500 KBit/s.'
-            ' By default, youtube-dl will pick the best quality.'
-            ' Use commas to download multiple audio formats, such as'
-            ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
-            ' You can merge the video and audio of two formats into a single'
-            ' file using -f <video-format>+<audio-format> (requires ffmpeg or'
-            ' avconv), for example -f bestvideo+bestaudio.'))
+        help='Video format code, see the "FORMAT SELECTION" for all the info')
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',

From 7b071e317bd79c81fac0c529d021da381801c3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Apr 2015 18:52:01 +0200
Subject: [PATCH 0308/1515] README: document bestvideo+bestaudio/best (#5447)

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 68426b685..7dbe66995 100644
--- a/README.md
+++ b/README.md
@@ -270,6 +270,8 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
+youtube-dl uses `-f bestvideo+bestaudio/best` if ffmpeg or avconv are installed (`best` is needed for videos that don't come from YouTube because they don't provide the audio and video in two different files). If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+
 # VIDEO SELECTION
 
 Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:

From b524a001d68767298272ef59274c826a6ea6a1b7 Mon Sep 17 00:00:00 2001
From: Quentin Rameau <quinq@quinq.eu.org>
Date: Mon, 20 Apr 2015 06:25:55 +0200
Subject: [PATCH 0309/1515] [bandcamp] fix video_id parsing (fixes #4861)

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 869294967..505877b77 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -72,7 +72,7 @@ class BandcampIE(InfoExtractor):
 
         download_link = m_download.group(1)
         video_id = self._search_regex(
-            r'(?ms)var TralbumData = {.*?id: (?P<id>\d+),?$',
+            r'(?ms)var TralbumData = .*?[{,]\s*id: (?P<id>\d+),?$',
             webpage, 'video id')
 
         download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')

From 3ded7bac166c71f99b8d8036c072552c483d2364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:13:31 +0600
Subject: [PATCH 0310/1515] [extractor/common] Add ability to specify custom
 field preference for `_sort_formats`

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7757bf950..71230323c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -708,7 +708,7 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
-    def _sort_formats(self, formats):
+    def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')
 
@@ -718,6 +718,9 @@ class InfoExtractor(object):
             if not f.get('ext') and 'url' in f:
                 f['ext'] = determine_ext(f['url'])
 
+            if isinstance(field_preference, (list, tuple)):
+                return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
+
             preference = f.get('preference')
             if preference is None:
                 proto = f.get('protocol')

From 736785ab63104b4d285545e4755fa03ef4496a47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:42:20 +0600
Subject: [PATCH 0311/1515] [ted] Clarify audio/video-only formats

---
 youtube_dl/extractor/ted.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index a2dc14c2b..2c8acfef6 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -194,14 +194,18 @@ class TEDIE(InfoExtractor):
                         'tbr': int_or_none(resource.get('bitrate')),
                     })
             elif format_id == 'hls':
-                formats.extend(self._extract_m3u8_formats(
-                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id))
+                hls_formats = self._extract_m3u8_formats(
+                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
+                for f in hls_formats:
+                    f['acodec'] = 'none'
+                formats.extend(hls_formats)
 
         audio_download = talk_info.get('audioDownload')
         if audio_download:
             formats.append({
                 'url': audio_download,
                 'format_id': 'audio',
+                'vcodec': 'none',
             })
 
         self._sort_formats(formats)

From cfbee8a431fdc22ff98dc115a59e0a48cace2c0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:42:42 +0600
Subject: [PATCH 0312/1515] [ted] Clarify IE_NAME

---
 youtube_dl/extractor/ted.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 2c8acfef6..e2cbbc121 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -10,6 +10,7 @@ from ..utils import int_or_none
 
 
 class TEDIE(InfoExtractor):
+    IE_NAME = 'ted'
     _VALID_URL = r'''(?x)
         (?P<proto>https?://)
         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/

From 17c8675853f679d3671c1558f6b2f060f2c8e23a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:58:29 +0600
Subject: [PATCH 0313/1515] [YoutubeDL] Allow bestvideo+bestaudio/best strategy
 for ted extractor

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3bb350e2a..b5fddb8e7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1092,7 +1092,7 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if info_dict['extractor'] == 'youtube' and FFmpegMergerPP(self).available:
+            if info_dict['extractor'] in ['youtube', 'ted'] and FFmpegMergerPP(self).available:
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 6728187ac0b1d5d083e8654e35a18d33485f2614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:58:46 +0600
Subject: [PATCH 0314/1515] [YoutubeDL] mp3 is compatible with mp4

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5fddb8e7..5b2c3aa38 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1380,7 +1380,7 @@ class YoutubeDL(object):
                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
                         if video_ext and audio_ext:
                             COMPATIBLE_EXTS = (
-                                ('mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
+                                ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
                                 ('webm')
                             )
                             for exts in COMPATIBLE_EXTS:

From 0f0b5736dab5e45189d1b35e0b7074585edd057b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:01:02 +0600
Subject: [PATCH 0315/1515] [ted] Fix hls audio/video-only formats

---
 youtube_dl/extractor/ted.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index e2cbbc121..c788feb5d 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -198,7 +198,10 @@ class TEDIE(InfoExtractor):
                 hls_formats = self._extract_m3u8_formats(
                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
                 for f in hls_formats:
-                    f['acodec'] = 'none'
+                    if not f.get('height'):
+                        f['vcodec'] = 'none'
+                    else:
+                        f['acodec'] = 'none'
                 formats.extend(hls_formats)
 
         audio_download = talk_info.get('audioDownload')

From 14f7abfa71bc71f46a5e3306062b1feccc4ccd26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:04:17 +0600
Subject: [PATCH 0316/1515] [ted] Lower preference for direct audio since it's
 mono

---
 youtube_dl/extractor/ted.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index c788feb5d..8aa27d282 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -210,6 +210,7 @@ class TEDIE(InfoExtractor):
                 'url': audio_download,
                 'format_id': 'audio',
                 'vcodec': 'none',
+                'preference': -0.5,
             })
 
         self._sort_formats(formats)

From 6621ca39a3f03e44648b54bd8ff57814362f5dbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:04:42 +0600
Subject: [PATCH 0317/1515] [ted] Skip hls quality selection format

---
 youtube_dl/extractor/ted.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 8aa27d282..a48d77c30 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -198,6 +198,8 @@ class TEDIE(InfoExtractor):
                 hls_formats = self._extract_m3u8_formats(
                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
                 for f in hls_formats:
+                    if f.get('format_id') == 'hls-meta':
+                        continue
                     if not f.get('height'):
                         f['vcodec'] = 'none'
                     else:

From bda44f31a1e18d71dc1cc7a2a3a754593ecd63d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:33:35 +0600
Subject: [PATCH 0318/1515] [bambuser] Modernize

---
 youtube_dl/extractor/bambuser.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index c193e66ca..12673e766 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -35,12 +35,11 @@ class BambuserIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        info_url = ('http://player-c.api.bambuser.com/getVideo.json?'
-                    '&api_key=%s&vid=%s' % (self._API_KEY, video_id))
-        info_json = self._download_webpage(info_url, video_id)
-        info = json.loads(info_json)['result']
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://player-c.api.bambuser.com/getVideo.json?api_key=%s&vid=%s'
+            % (self._API_KEY, video_id), video_id)['result']
 
         return {
             'id': video_id,

From ae8953409e30ef9d7dd6bc40350fccc2a29ffb4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:35:53 +0600
Subject: [PATCH 0319/1515] [bambuser] Capture and output error message (#5478)

---
 youtube_dl/extractor/bambuser.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 12673e766..d52302ebc 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class BambuserIE(InfoExtractor):
@@ -39,17 +37,24 @@ class BambuserIE(InfoExtractor):
 
         info = self._download_json(
             'http://player-c.api.bambuser.com/getVideo.json?api_key=%s&vid=%s'
-            % (self._API_KEY, video_id), video_id)['result']
+            % (self._API_KEY, video_id), video_id)
+
+        error = info.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
+
+        result = info['result']
 
         return {
             'id': video_id,
-            'title': info['title'],
-            'url': info['url'],
-            'thumbnail': info.get('preview'),
-            'duration': int(info['length']),
-            'view_count': int(info['views_total']),
-            'uploader': info['username'],
-            'uploader_id': info['owner']['uid'],
+            'title': result['title'],
+            'url': result['url'],
+            'thumbnail': result.get('preview'),
+            'duration': int(result['length']),
+            'view_count': int(result['views_total']),
+            'uploader': result['username'],
+            'uploader_id': result['owner']['uid'],
         }
 
 

From edf421611928e48ac4fda778d6de8ea83585019d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:46:01 +0600
Subject: [PATCH 0320/1515] [bambuser] Modernize and extract more metadata

---
 youtube_dl/extractor/bambuser.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index d52302ebc..93913c3f4 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -4,8 +4,15 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..compat import (
+    compat_urllib_request,
+    compat_str,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+)
 
 
 class BambuserIE(InfoExtractor):
@@ -24,6 +31,9 @@ class BambuserIE(InfoExtractor):
             'duration': 3741,
             'uploader': 'pixelversity',
             'uploader_id': '344706',
+            'timestamp': 1382976692,
+            'upload_date': '20131028',
+            'view_count': int,
         },
         'params': {
             # It doesn't respect the 'Range' header, it would download the whole video
@@ -51,10 +61,13 @@ class BambuserIE(InfoExtractor):
             'title': result['title'],
             'url': result['url'],
             'thumbnail': result.get('preview'),
-            'duration': int(result['length']),
-            'view_count': int(result['views_total']),
-            'uploader': result['username'],
-            'uploader_id': result['owner']['uid'],
+            'duration': int_or_none(result.get('length')),
+            'uploader': result.get('username'),
+            'uploader_id': compat_str(result.get('owner', {}).get('uid')),
+            'timestamp': int_or_none(result.get('created')),
+            'fps': float_or_none(result.get('framerate')),
+            'view_count': int_or_none(result.get('views_total')),
+            'comment_count': int_or_none(result.get('comment_count')),
         }
 
 

From 006ce15a0cfbf79dedd2c59b1e17344b1a130e81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 23:00:37 +0600
Subject: [PATCH 0321/1515] [bambuser] Add support for authentication (#5478)

---
 youtube_dl/extractor/bambuser.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 93913c3f4..8dff1d6e3 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -5,6 +5,7 @@ import itertools
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse,
     compat_urllib_request,
     compat_str,
 )
@@ -19,6 +20,8 @@ class BambuserIE(InfoExtractor):
     IE_NAME = 'bambuser'
     _VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
     _API_KEY = '005f64509e19a868399060af746a00aa'
+    _LOGIN_URL = 'https://bambuser.com/user'
+    _NETRC_MACHINE = 'bambuser'
 
     _TEST = {
         'url': 'http://bambuser.com/v/4050584',
@@ -42,6 +45,34 @@ class BambuserIE(InfoExtractor):
         },
     }
 
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'form_id': 'user_login',
+            'op': 'Log in',
+            'name': username,
+            'pass': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Referer', self._LOGIN_URL)
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        login_error = self._html_search_regex(
+            r'(?s)<div class="messages error">(.+?)</div>',
+            response, 'login error', default=None)
+        if login_error:
+            raise ExtractorError(
+                'Unable to login: %s' % login_error, expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From c6391cd587a26eb3d9bba7296be804f14612e919 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 02:29:56 +0800
Subject: [PATCH 0322/1515] [Senate] Add new extractor (#5302)

---
 youtube_dl/downloader/f4m.py       |   2 +
 youtube_dl/extractor/__init__.py   |   1 +
 youtube_dl/extractor/senateisvp.py | 129 +++++++++++++++++++++++++++++
 3 files changed, 132 insertions(+)
 create mode 100644 youtube_dl/extractor/senateisvp.py

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 4ab000d67..b1a858c45 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -389,6 +389,8 @@ class F4mFD(FileDownloader):
             url = base_url + name
             if akamai_pv:
                 url += '?' + akamai_pv.strip(';')
+            if info_dict.get('extra_param_to_segment_url'):
+                url += info_dict.get('extra_param_to_segment_url')
             frag_filename = '%s-%s' % (tmpfilename, name)
             try:
                 success = http_dl.download(frag_filename, {'url': url})
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1dabea7d6..3bd6d1697 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -447,6 +447,7 @@ from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
 from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
+from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
new file mode 100644
index 000000000..807979d13
--- /dev/null
+++ b/youtube_dl/extractor/senateisvp.py
@@ -0,0 +1,129 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import ExtractorError
+from ..compat import (
+    compat_parse_qs,
+    compat_urlparse,
+)
+
+
+class SenateISVPIE(InfoExtractor):
+    _COMM_MAP = [
+        ["ag", "76440", "http://ag-f.akamaihd.net"],
+        ["aging", "76442", "http://aging-f.akamaihd.net"],
+        ["approps", "76441", "http://approps-f.akamaihd.net"],
+        ["armed", "76445", "http://armed-f.akamaihd.net"],
+        ["banking", "76446", "http://banking-f.akamaihd.net"],
+        ["budget", "76447", "http://budget-f.akamaihd.net"],
+        ["cecc", "76486", "http://srs-f.akamaihd.net"],
+        ["commerce", "80177", "http://commerce1-f.akamaihd.net"],
+        ["csce", "75229", "http://srs-f.akamaihd.net"],
+        ["dpc", "76590", "http://dpc-f.akamaihd.net"],
+        ["energy", "76448", "http://energy-f.akamaihd.net"],
+        ["epw", "76478", "http://epw-f.akamaihd.net"],
+        ["ethics", "76449", "http://ethics-f.akamaihd.net"],
+        ["finance", "76450", "http://finance-f.akamaihd.net"],
+        ["foreign", "76451", "http://foreign-f.akamaihd.net"],
+        ["govtaff", "76453", "http://govtaff-f.akamaihd.net"],
+        ["help", "76452", "http://help-f.akamaihd.net"],
+        ["indian", "76455", "http://indian-f.akamaihd.net"],
+        ["intel", "76456", "http://intel-f.akamaihd.net"],
+        ["intlnarc", "76457", "http://intlnarc-f.akamaihd.net"],
+        ["jccic", "85180", "http://jccic-f.akamaihd.net"],
+        ["jec", "76458", "http://jec-f.akamaihd.net"],
+        ["judiciary", "76459", "http://judiciary-f.akamaihd.net"],
+        ["rpc", "76591", "http://rpc-f.akamaihd.net"],
+        ["rules", "76460", "http://rules-f.akamaihd.net"],
+        ["saa", "76489", "http://srs-f.akamaihd.net"],
+        ["smbiz", "76461", "http://smbiz-f.akamaihd.net"],
+        ["srs", "75229", "http://srs-f.akamaihd.net"],
+        ["uscc", "76487", "http://srs-f.akamaihd.net"],
+        ["vetaff", "76462", "http://vetaff-f.akamaihd.net"],
+        ["arch", "", "http://ussenate-f.akamaihd.net/"]
+    ]
+    _IE_NAME = 'senate.gov'
+    _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
+    _TESTS = [{
+        'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
+        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
+        'info_dict': {
+            'id': 'judiciary031715',
+            'ext': 'flv',
+            'title': 'Integrated Senate Video Player',
+        }
+    }, {
+        'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
+        'md5': '2917c827513700aa9b70eaebf25116da',
+        'info_dict': {
+            'id': 'commerce011514',
+            'ext': 'flv',
+            'title': 'Integrated Senate Video Player'
+        }
+    }, {
+        'url': 'http://www.senate.gov/isvp/?type=arch&comm=intel&filename=intel090613&hc_location=ufi',
+        # checksum differs each time
+        'info_dict': {
+            'id': 'intel090613',
+            'ext': 'mp4',
+            'title': 'Integrated Senate Video Player'
+        }
+    }]
+
+    def _get_info_for_comm(self, committee):
+        for entry in self._COMM_MAP:
+            if entry[0] == committee:
+                return entry[1:]
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('qs'))
+        if not qs.get('filename') or not qs.get('type') or not qs.get('comm'):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        video_id = re.sub(r'.mp4$', '', qs['filename'][0])
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
+
+        video_type = qs['type'][0]
+        committee = video_type if video_type == 'arch' else qs['comm'][0]
+        stream_num, domain = self._get_info_for_comm(committee)
+
+        formats = []
+        if video_type == 'arch':
+            filename = video_id if '.' in video_id else video_id + '.mp4'
+            formats = [{
+                # All parameters in the query string are necessary to prevent a 403 error
+                'url': compat_urlparse.urljoin(domain, filename) + '?v=3.1.0&fp=&r=&g=',
+            }]
+        else:
+            hdcore_sign = '?hdcore=3.1.0'
+            url_params = (domain, video_id, stream_num)
+            f4m_url = '%s/z/%s_1@%s/manifest.f4m' % url_params + hdcore_sign
+            m3u8_url = '%s/i/%s_1@%s/master.m3u8' % url_params
+            for entry in self._extract_f4m_formats(f4m_url, video_id, f4m_id='f4m'):
+                # URLs without the extra param induce an 404 error
+                entry.update({'extra_param_to_segment_url': hdcore_sign})
+                formats.append(entry)
+            for entry in self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', m3u8_id='m3u8'):
+                mobj = re.search(r'(?P<tag>(?:-p|-b)).m3u8', entry['url'])
+                if mobj:
+                    entry['format_id'] += mobj.group('tag')
+                formats.append(entry)
+
+            self._sort_formats(formats)
+
+        info_dict = {
+            'id': video_id,
+            'title': title,
+        }
+
+        if len(formats) >= 1:
+            info_dict.update({'formats': formats})
+        else:
+            info_dict.update(formats[0])
+
+        return info_dict

From 24e21613b670efefd2f284c8e7027023d0a64399 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 02:32:10 +0800
Subject: [PATCH 0323/1515] [bilibili] Capture the video-not-exist message

---
 youtube_dl/extractor/bilibili.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 75d744852..904d9a8b4 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    ExtractorError,
 )
 
 
@@ -30,6 +31,8 @@ class BiliBiliIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if self._search_regex(r'(此视频不存在或被删除)', webpage, 'error message', default=None):
+            raise ExtractorError('The video does not exist or was deleted', expected=True)
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 

From f91e1a8739a59bca1ced0bbc70f8cf9c3a33f778 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 02:57:32 +0800
Subject: [PATCH 0324/1515] [Senate] Try to capture thumbnails

---
 youtube_dl/extractor/senateisvp.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 807979d13..a93874cad 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -53,6 +53,7 @@ class SenateISVPIE(InfoExtractor):
             'id': 'judiciary031715',
             'ext': 'flv',
             'title': 'Integrated Senate Video Player',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)$',
         }
     }, {
         'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
@@ -87,6 +88,9 @@ class SenateISVPIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
+        poster = qs.get('poster')
+        if poster:
+            thumbnail = poster[0]
 
         video_type = qs['type'][0]
         committee = video_type if video_type == 'arch' else qs['comm'][0]
@@ -119,6 +123,7 @@ class SenateISVPIE(InfoExtractor):
         info_dict = {
             'id': video_id,
             'title': title,
+            'thumbnail': thumbnail,
         }
 
         if len(formats) >= 1:

From 2fe1b5bd2add12d70717878704cd3f811af5d22c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 03:18:38 +0800
Subject: [PATCH 0325/1515] [CSpan] Add detection for Senate ISVP. Closes #5302

---
 youtube_dl/extractor/cspan.py      | 18 +++++++++++++++++-
 youtube_dl/extractor/generic.py    |  6 ++++++
 youtube_dl/extractor/senateisvp.py | 20 ++++++++++++++++++--
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 955119d40..7377ac7b9 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -7,7 +7,9 @@ from ..utils import (
     int_or_none,
     unescapeHTML,
     find_xpath_attr,
+    smuggle_url,
 )
+from .senateisvp import SenateISVPIE
 
 
 class CSpanIE(InfoExtractor):
@@ -40,6 +42,15 @@ class CSpanIE(InfoExtractor):
             'title': 'General Motors Ignition Switch Recall',
         },
         'playlist_duration_sum': 14855,
+    }, {
+        # Video from senate.gov
+        'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
+        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
+        'info_dict': {
+            'id': 'judiciary031715',
+            'ext': 'flv',
+            'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
+        }
     }]
 
     def _real_extract(self, url):
@@ -56,7 +67,7 @@ class CSpanIE(InfoExtractor):
                 # present, otherwise this is a stripped version
                 r'<p class=\'initial\'>(.*?)</p>'
             ],
-            webpage, 'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL, default=None)
 
         info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
         data = self._download_json(info_url, video_id)
@@ -68,6 +79,11 @@ class CSpanIE(InfoExtractor):
         title = find_xpath_attr(doc, './/string', 'name', 'title').text
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
+        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+        if senate_isvp_url:
+            surl = smuggle_url(senate_isvp_url, {'force_title': title})
+            return self.url_result(surl, 'SenateISVP', video_id, title)
+
         files = data['video']['files']
 
         entries = [{
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e645d1bb3..ec4d0c210 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -35,6 +35,7 @@ from .rutv import RUTVIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
+from .senateisvp import SenateISVPIE
 
 
 class GenericIE(InfoExtractor):
@@ -1365,6 +1366,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
 
+        # Look for Senate ISVP iframe
+        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+        if senate_isvp_url:
+            return self.url_result(surl, 'SenateISVP')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index a93874cad..23e1cd944 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 
 import re
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    unsmuggle_url,
+)
 from ..compat import (
     compat_parse_qs,
     compat_urlparse,
@@ -73,12 +76,22 @@ class SenateISVPIE(InfoExtractor):
         }
     }]
 
+    @staticmethod
+    def _search_iframe_url(webpage):
+        mobj = re.search(
+            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/\?[^'\"]+)['\"]",
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _get_info_for_comm(self, committee):
         for entry in self._COMM_MAP:
             if entry[0] == committee:
                 return entry[1:]
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         qs = compat_parse_qs(re.match(self._VALID_URL, url).group('qs'))
         if not qs.get('filename') or not qs.get('type') or not qs.get('comm'):
             raise ExtractorError('Invalid URL', expected=True)
@@ -87,7 +100,10 @@ class SenateISVPIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
+        if smuggled_data.get('force_title'):
+            title = smuggled_data['force_title']
+        else:
+            title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
         poster = qs.get('poster')
         if poster:
             thumbnail = poster[0]

From 92dcba1e1cc411eecb64792e78ad8f3b125691c8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 03:30:54 +0800
Subject: [PATCH 0326/1515] [CSpan] Fix test cases CSpan_1 and CSpan_2

---
 youtube_dl/extractor/cspan.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 7377ac7b9..6bebcc65c 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -37,11 +37,14 @@ class CSpanIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
+        'md5': '446562a736c6bf97118e389433ed88d4',
         'info_dict': {
             'id': '342759',
+            'ext': 'mp4',
             'title': 'General Motors Ignition Switch Recall',
+            'duration': 14848,
+            'description': 'md5:70c7c3b8fa63fa60d42772440596034c'
         },
-        'playlist_duration_sum': 14855,
     }, {
         # Video from senate.gov
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
@@ -97,9 +100,14 @@ class CSpanIE(InfoExtractor):
             'duration': int_or_none(f.get('length', {}).get('#text')),
         } for partnum, f in enumerate(files)]
 
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'title': title,
-            'id': video_id,
-        }
+        if len(entries) == 1:
+            entry = dict(entries[0])
+            entry['id'] = video_id
+            return entry
+        else:
+            return {
+                '_type': 'playlist',
+                'entries': entries,
+                'title': title,
+                'id': video_id,
+            }

From 13a11b195ff00648ab5631db5b1ae7c46548c7fb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 05:13:25 +0800
Subject: [PATCH 0327/1515] [SenateISVP] Fix tests

Remove md5 sums. They differs from my PC and the travis worker.
---
 youtube_dl/extractor/senateisvp.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 23e1cd944..fa6610261 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -51,7 +51,6 @@ class SenateISVPIE(InfoExtractor):
     _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
-        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
         'info_dict': {
             'id': 'judiciary031715',
             'ext': 'flv',
@@ -60,7 +59,6 @@ class SenateISVPIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
-        'md5': '2917c827513700aa9b70eaebf25116da',
         'info_dict': {
             'id': 'commerce011514',
             'ext': 'flv',
@@ -105,8 +103,7 @@ class SenateISVPIE(InfoExtractor):
         else:
             title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
         poster = qs.get('poster')
-        if poster:
-            thumbnail = poster[0]
+        thumbnail = poster[0] if poster else None
 
         video_type = qs['type'][0]
         committee = video_type if video_type == 'arch' else qs['comm'][0]

From da55dac047d63d48fd12247369b2fb858c4210ef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 05:22:23 +0800
Subject: [PATCH 0328/1515] [CSpan] Removed the md5 sum of CSpan_3

---
 youtube_dl/extractor/cspan.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 6bebcc65c..d516b1402 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -48,7 +48,6 @@ class CSpanIE(InfoExtractor):
     }, {
         # Video from senate.gov
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
-        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
         'info_dict': {
             'id': 'judiciary031715',
             'ext': 'flv',

From 0954cd8aa4421b1844a30e99702c364f1fffc15f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 13:48:02 +0800
Subject: [PATCH 0329/1515] [Cinemassacre] Add detection for videos from
 blip.tv

---
 youtube_dl/extractor/bliptv.py          |  9 +++++++++
 youtube_dl/extractor/generic.py         | 10 ++++------
 youtube_dl/extractor/screenwavemedia.py | 23 ++++++++++++++++++++++-
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index b632ce967..fb56cd78d 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -102,6 +102,15 @@ class BlipTVIE(InfoExtractor):
         },
     ]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
+        if mobj:
+            return 'http://blip.tv/a/a-' + mobj.group(1)
+        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
+        if mobj:
+            return mobj.group(1)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         lookup_id = mobj.group('lookup_id')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec4d0c210..4946cc132 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,6 +36,7 @@ from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
+from .bliptv import BlipTVIE
 
 
 class GenericIE(InfoExtractor):
@@ -1073,12 +1074,9 @@ class GenericIE(InfoExtractor):
             }
 
         # Look for embedded blip.tv player
-        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
-        if mobj:
-            return self.url_result('http://blip.tv/a/a-' + mobj.group(1), 'BlipTV')
-        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
-        if mobj:
-            return self.url_result(mobj.group(1), 'BlipTV')
+        bliptv_url = BlipTVIE._extract_url(webpage)
+        if bliptv_url:
+            return self.url_result(bliptv_url, 'BlipTV')
 
         # Look for embedded condenast player
         matches = re.findall(
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 6c9fdb7c1..b515b11b4 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -7,7 +7,9 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    ExtractorError
 )
+from .bliptv import BlipTVIE
 
 
 class ScreenwaveMediaIE(InfoExtractor):
@@ -104,6 +106,20 @@ class CinemassacreIE(InfoExtractor):
                 'upload_date': '20131002',
                 'title': 'The Mummy’s Hand (1940)',
             },
+        },
+        {
+            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
+            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
+            'info_dict': {
+                'id': '4065369',
+                'ext': 'flv',
+                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
+                'upload_date': '20061207',
+                'uploader': 'cinemassacre',
+                'uploader_id': '250778',
+                'timestamp': 1283233867,
+                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
+            }
         }
     ]
 
@@ -116,7 +132,12 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
-            webpage, 'player data URL')
+            webpage, 'player data URL', default=None)
+        if not playerdata_url:
+            playerdata_url = BlipTVIE._extract_url(webpage)
+        if not playerdata_url:
+            raise ExtractorError('Unable to find player data')
+
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
         video_description = self._html_search_regex(

From e94443de80d20a62fccda23fa25abcfa3798243a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 15:10:27 +0800
Subject: [PATCH 0330/1515] [Cinemassacre] Move to a standalone module

---
 youtube_dl/extractor/__init__.py        |  3 +-
 youtube_dl/extractor/cinemassacre.py    | 81 +++++++++++++++++++++++++
 youtube_dl/extractor/screenwavemedia.py | 75 -----------------------
 3 files changed, 83 insertions(+), 76 deletions(-)
 create mode 100644 youtube_dl/extractor/cinemassacre.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3bd6d1697..a64afa1da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -70,6 +70,7 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
+from .cinemassacre import CinemassacreIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
@@ -446,7 +447,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
-from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
+from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
new file mode 100644
index 000000000..c8bbebe1a
--- /dev/null
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -0,0 +1,81 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+from .bliptv import BlipTVIE
+
+
+class CinemassacreIE(InfoExtractor):
+    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
+    _TESTS = [
+        {
+            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
+            'md5': 'fde81fbafaee331785f58cd6c0d46190',
+            'info_dict': {
+                'id': 'Cinemassacre-19911',
+                'ext': 'mp4',
+                'upload_date': '20121110',
+                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
+                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
+            },
+        },
+        {
+            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
+            'md5': 'd72f10cd39eac4215048f62ab477a511',
+            'info_dict': {
+                'id': 'Cinemassacre-521be8ef82b16',
+                'ext': 'mp4',
+                'upload_date': '20131002',
+                'title': 'The Mummy’s Hand (1940)',
+            },
+        },
+        {
+            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
+            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
+            'info_dict': {
+                'id': '4065369',
+                'ext': 'flv',
+                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
+                'upload_date': '20061207',
+                'uploader': 'cinemassacre',
+                'uploader_id': '250778',
+                'timestamp': 1283233867,
+                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
+
+        webpage = self._download_webpage(url, display_id)
+
+        playerdata_url = self._search_regex(
+            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            webpage, 'player data URL', default=None)
+        if not playerdata_url:
+            playerdata_url = BlipTVIE._extract_url(webpage)
+        if not playerdata_url:
+            raise ExtractorError('Unable to find player data')
+
+        video_title = self._html_search_regex(
+            r'<title>(?P<title>.+?)\|', webpage, 'title')
+        video_description = self._html_search_regex(
+            r'<div class="entry-content">(?P<description>.+?)</div>',
+            webpage, 'description', flags=re.DOTALL, fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            '_type': 'url_transparent',
+            'display_id': display_id,
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            'url': playerdata_url,
+        }
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index b515b11b4..74fb1983a 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -7,9 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
-    ExtractorError
 )
-from .bliptv import BlipTVIE
 
 
 class ScreenwaveMediaIE(InfoExtractor):
@@ -83,79 +81,6 @@ class ScreenwaveMediaIE(InfoExtractor):
         }
 
 
-class CinemassacreIE(InfoExtractor):
-    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
-    _TESTS = [
-        {
-            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
-            'md5': 'fde81fbafaee331785f58cd6c0d46190',
-            'info_dict': {
-                'id': 'Cinemassacre-19911',
-                'ext': 'mp4',
-                'upload_date': '20121110',
-                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
-                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
-            'md5': 'd72f10cd39eac4215048f62ab477a511',
-            'info_dict': {
-                'id': 'Cinemassacre-521be8ef82b16',
-                'ext': 'mp4',
-                'upload_date': '20131002',
-                'title': 'The Mummy’s Hand (1940)',
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
-            'info_dict': {
-                'id': '4065369',
-                'ext': 'flv',
-                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
-                'upload_date': '20061207',
-                'uploader': 'cinemassacre',
-                'uploader_id': '250778',
-                'timestamp': 1283233867,
-                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
-            }
-        }
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
-
-        webpage = self._download_webpage(url, display_id)
-
-        playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
-            webpage, 'player data URL', default=None)
-        if not playerdata_url:
-            playerdata_url = BlipTVIE._extract_url(webpage)
-        if not playerdata_url:
-            raise ExtractorError('Unable to find player data')
-
-        video_title = self._html_search_regex(
-            r'<title>(?P<title>.+?)\|', webpage, 'title')
-        video_description = self._html_search_regex(
-            r'<div class="entry-content">(?P<description>.+?)</div>',
-            webpage, 'description', flags=re.DOTALL, fatal=False)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            '_type': 'url_transparent',
-            'display_id': display_id,
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            'url': playerdata_url,
-        }
-
-
 class TeamFourIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/video/(?P<id>[a-z0-9\-]+)/?'
     _TEST = {

From 757cda0a96e5efd0c1fb86824a99e558a7c797b0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 15:20:40 +0800
Subject: [PATCH 0331/1515] [Cinemassacre] Support Youtube embedded videos
 (fixes #5131)

---
 youtube_dl/extractor/cinemassacre.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index c8bbebe1a..cf0a7551b 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -33,6 +33,7 @@ class CinemassacreIE(InfoExtractor):
             },
         },
         {
+            # blip.tv embedded video
             'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
             'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
             'info_dict': {
@@ -45,6 +46,20 @@ class CinemassacreIE(InfoExtractor):
                 'timestamp': 1283233867,
                 'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
             }
+        },
+        {
+            # Youtube embedded video
+            'url': 'http://cinemassacre.com/2006/09/01/mckids/',
+            'md5': '6eb30961fa795fedc750eac4881ad2e1',
+            'info_dict': {
+                'id': 'FnxsNhuikpo',
+                'ext': 'mp4',
+                'upload_date': '20060901',
+                'uploader': 'Cinemassacre Extras',
+                'description': 'md5:de9b751efa9e45fbaafd9c8a1123ed53',
+                'uploader_id': 'Cinemassacre',
+                'title': 'AVGN: McKids',
+            }
         }
     ]
 
@@ -56,7 +71,10 @@ class CinemassacreIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            [
+                r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+                r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
+            ],
             webpage, 'player data URL', default=None)
         if not playerdata_url:
             playerdata_url = BlipTVIE._extract_url(webpage)

From 5c1e6f69c421c244139b1f4c6d2759dadfbc1abd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Apr 2015 15:04:55 +0200
Subject: [PATCH 0332/1515] [senate] Simplify

There isn't any problem if the 'formats' field only has one element
---
 youtube_dl/extractor/senateisvp.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index fa6610261..d3b8a1be4 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -133,15 +133,9 @@ class SenateISVPIE(InfoExtractor):
 
             self._sort_formats(formats)
 
-        info_dict = {
+        return {
             'id': video_id,
             'title': title,
+            'formats': formats,
             'thumbnail': thumbnail,
         }
-
-        if len(formats) >= 1:
-            info_dict.update({'formats': formats})
-        else:
-            info_dict.update(formats[0])
-
-        return info_dict

From ed553379dfd4d564f8335defc1067eeecd536f04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 20:55:05 +0600
Subject: [PATCH 0333/1515] [youtube:ytsearch] Temporary workaround (#5483)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 52909b0da..4ec39c589 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1517,7 +1517,7 @@ class YoutubeSearchIE(SearchInfoExtractor):
         while (PAGE_SIZE * pagenum) < limit:
             result_url = self._API_URL % (
                 compat_urllib_parse.quote_plus(query.encode('utf-8')),
-                (PAGE_SIZE * pagenum) + 1)
+                max((PAGE_SIZE * pagenum) + 1), 2)
             data_json = self._download_webpage(
                 result_url, video_id='query "%s"' % query,
                 note='Downloading page %s' % (pagenum + 1),

From eb0f3e7ec080549c1df6a104fc59400efd9a992a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:36:41 +0600
Subject: [PATCH 0334/1515] [youtube:user] Extract in terms of
 `load_more_widget_html`

---
 youtube_dl/extractor/youtube.py | 49 +++------------------------------
 1 file changed, 4 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4ec39c589..0b4038038 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1358,6 +1358,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
+    _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
     IE_NAME = 'youtube:channel'
     _TESTS = [{
         'note': 'paginated channel',
@@ -1386,7 +1387,7 @@ class YoutubeChannelIE(InfoExtractor):
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        url = 'https://www.youtube.com/channel/%s/videos' % channel_id
+        url = self._TEMPLATE_URL % channel_id
         channel_page = self._download_webpage(url, channel_id)
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:
@@ -1429,12 +1430,10 @@ class YoutubeChannelIE(InfoExtractor):
         return self.playlist_result(_entries(), channel_id)
 
 
-class YoutubeUserIE(InfoExtractor):
+class YoutubeUserIE(YoutubeChannelIE):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
-    _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
-    _GDATA_PAGE_SIZE = 50
-    _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
+    _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
     IE_NAME = 'youtube:user'
 
     _TESTS = [{
@@ -1458,46 +1457,6 @@ class YoutubeUserIE(InfoExtractor):
         else:
             return super(YoutubeUserIE, cls).suitable(url)
 
-    def _real_extract(self, url):
-        username = self._match_id(url)
-
-        # Download video ids using YouTube Data API. Result size per
-        # query is limited (currently to 50 videos) so we need to query
-        # page by page until there are no video ids - it means we got
-        # all of them.
-
-        def download_page(pagenum):
-            start_index = pagenum * self._GDATA_PAGE_SIZE + 1
-
-            gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
-            page = self._download_webpage(
-                gdata_url, username,
-                'Downloading video ids from %d to %d' % (
-                    start_index, start_index + self._GDATA_PAGE_SIZE))
-
-            try:
-                response = json.loads(page)
-            except ValueError as err:
-                raise ExtractorError('Invalid JSON in API response: ' + compat_str(err))
-            if 'entry' not in response['feed']:
-                return
-
-            # Extract video identifiers
-            entries = response['feed']['entry']
-            for entry in entries:
-                title = entry['title']['$t']
-                video_id = entry['id']['$t'].split('/')[-1]
-                yield {
-                    '_type': 'url',
-                    'url': video_id,
-                    'ie_key': 'Youtube',
-                    'id': video_id,
-                    'title': title,
-                }
-        url_results = OnDemandPagedList(download_page, self._GDATA_PAGE_SIZE)
-
-        return self.playlist_result(url_results, playlist_title=username)
-
 
 class YoutubeSearchIE(SearchInfoExtractor):
     IE_DESC = 'YouTube.com searches'

From 60bf45c80d377a38b00b9ec1426c4cc1d9003742 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:37:45 +0600
Subject: [PATCH 0335/1515] [youtube:channel] Specify first page download
 message

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0b4038038..1469b932f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1388,7 +1388,7 @@ class YoutubeChannelIE(InfoExtractor):
         channel_id = self._match_id(url)
 
         url = self._TEMPLATE_URL % channel_id
-        channel_page = self._download_webpage(url, channel_id)
+        channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:
                     channel-header-autogenerated-label|

From 6de5dbafeebd2c54670e4cbf2833f9f10f3c2032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:42:21 +0600
Subject: [PATCH 0336/1515] [youtube:channel] Make `extract_videos_from_page`
 static

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1469b932f..8bf54931c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1369,7 +1369,8 @@ class YoutubeChannelIE(InfoExtractor):
         }
     }]
 
-    def extract_videos_from_page(self, page):
+    @staticmethod
+    def extract_videos_from_page(page):
         ids_in_page = []
         titles_in_page = []
         for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):

From cc38fa6cfbdab2ca77ecb1155d64574ab0004bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:55:59 +0600
Subject: [PATCH 0337/1515] [youtube] Remove unused import

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8bf54931c..b2ae08418 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,7 +28,6 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
-    OnDemandPagedList,
     orderedSet,
     unescapeHTML,
     unified_strdate,

From b4c08069638896e6c565536d52ab8fc9226c91bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Apr 2015 19:30:31 +0200
Subject: [PATCH 0338/1515] [youtube:ytsearch] Use the same system as the
 search webpage (fixes #5483)

The gdata api V2 was deprecated and according to http://youtube-eng.blogspot.com.es/2014/03/committing-to-youtube-data-api-v3_4.html remains available until April 20, 2015.
---
 youtube_dl/extractor/youtube.py | 51 +++++++++++++++++----------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b2ae08418..f1a5c0077 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1458,54 +1458,55 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
-class YoutubeSearchIE(SearchInfoExtractor):
+class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
     IE_DESC = 'YouTube.com searches'
-    _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
-    _MAX_RESULTS = 1000
+    # there doesn't appear to be a real limit, for example if you search for
+    # 'python' you get more than 8.000.000 results
+    _MAX_RESULTS = float('inf')
     IE_NAME = 'youtube:search'
     _SEARCH_KEY = 'ytsearch'
+    _EXTRA_QUERY_ARGS = {}
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
 
-        video_ids = []
-        pagenum = 0
+        videos = []
         limit = n
-        PAGE_SIZE = 50
 
-        while (PAGE_SIZE * pagenum) < limit:
-            result_url = self._API_URL % (
-                compat_urllib_parse.quote_plus(query.encode('utf-8')),
-                max((PAGE_SIZE * pagenum) + 1), 2)
-            data_json = self._download_webpage(
+        for pagenum in itertools.count(1):
+            url_query = {
+                'search_query': query,
+                'page': pagenum,
+                'spf': 'navigate',
+            }
+            url_query.update(self._EXTRA_QUERY_ARGS)
+            result_url = 'https://www.youtube.com/results?' + compat_urllib_parse.urlencode(url_query)
+            data = self._download_json(
                 result_url, video_id='query "%s"' % query,
-                note='Downloading page %s' % (pagenum + 1),
+                note='Downloading page %s' % pagenum,
                 errnote='Unable to download API page')
-            data = json.loads(data_json)
-            api_response = data['data']
+            html_content = data[1]['body']['content']
 
-            if 'items' not in api_response:
+            if 'class="search-message' in html_content:
                 raise ExtractorError(
                     '[youtube] No video results', expected=True)
 
-            new_ids = list(video['id'] for video in api_response['items'])
-            video_ids += new_ids
+            new_videos = self._ids_to_results(orderedSet(re.findall(
+                r'href="/watch\?v=(.{11})', html_content)))
+            videos += new_videos
+            if not new_videos or len(videos) > limit:
+                break
 
-            limit = min(n, api_response['totalItems'])
-            pagenum += 1
-
-        if len(video_ids) > n:
-            video_ids = video_ids[:n]
-        videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
-                  for video_id in video_ids]
+        if len(videos) > n:
+            videos = videos[:n]
         return self.playlist_result(videos, query)
 
 
 class YoutubeSearchDateIE(YoutubeSearchIE):
     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
-    _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
     _SEARCH_KEY = 'ytsearchdate'
     IE_DESC = 'YouTube.com searches, newest videos first'
+    _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
 
 
 class YoutubeSearchURLIE(InfoExtractor):

From 8be2bdfabd7a24936697f6681bd417de414a554a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Apr 2015 15:05:35 +0800
Subject: [PATCH 0339/1515] [YoutubeDL] Remove the redundant assignment to
 old_filename

Caused by commmit 592e97e8550389e22b716eb33c30584aa3a8d656
---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5b2c3aa38..0fdcf1b0b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1516,7 +1516,6 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
-            old_filename = info['filepath']
             try:
                 files_to_delete, info = pp.run(info)
             except PostProcessingError as e:

From 9dd8e46a2d0860421b4bb4f616f05e5ebd686380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 20:28:33 +0600
Subject: [PATCH 0340/1515] [youtube:search] Cancel out _TESTS

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f1a5c0077..07c0f6ef9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1466,6 +1466,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
     IE_NAME = 'youtube:search'
     _SEARCH_KEY = 'ytsearch'
     _EXTRA_QUERY_ARGS = {}
+    _TESTS = []
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""

From bc94bd510bfb8a98f7456912f930c69c21bd6e15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:01:25 +0600
Subject: [PATCH 0341/1515] [hitbox] Extract all formats (Closes #5494)

---
 youtube_dl/extractor/hitbox.py | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index d606429ca..d7cd8bb7e 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -96,13 +96,33 @@ class HitboxIE(InfoExtractor):
             'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
             video_id)
 
-        clip = player_config.get('clip')
-        video_url = clip.get('url')
-        res = clip.get('bitrates', [])[0].get('label')
+        formats = []
+        for video in player_config['clip']['bitrates']:
+            label = video.get('label')
+            if label == 'Auto':
+                continue
+            video_url = video.get('url')
+            if not video_url:
+                continue
+            bitrate = int_or_none(video.get('bitrate'))
+            if determine_ext(video_url) == 'm3u8':
+                if not video_url.startswith('http'):
+                    continue
+                formats.append({
+                    'url': video_url,
+                    'ext': 'mp4',
+                    'tbr': bitrate,
+                    'format_note': label,
+                    'protocol': 'm3u8_native',
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'tbr': bitrate,
+                    'format_note': label,
+                })
 
-        metadata['resolution'] = res
-        metadata['url'] = video_url
-        metadata['protocol'] = 'm3u8'
+        metadata['formats'] = formats
 
         return metadata
 

From 29492f33329c6d6fd4be7f2da7dcaff14bc531ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:01:52 +0600
Subject: [PATCH 0342/1515] [hitbox] Sort formats

---
 youtube_dl/extractor/hitbox.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index d7cd8bb7e..6701fa48d 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -121,6 +121,7 @@ class HitboxIE(InfoExtractor):
                     'tbr': bitrate,
                     'format_note': label,
                 })
+        self._sort_formats(formats)
 
         metadata['formats'] = formats
 

From 008bee0f50ea27e4fde4ba4d0cdb9043783299c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:03:56 +0600
Subject: [PATCH 0343/1515] [hitbox] Extract formats before metadata

---
 youtube_dl/extractor/hitbox.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 6701fa48d..c51bc7ca4 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -88,10 +88,6 @@ class HitboxIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/video',
-            video_id)
-
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
             video_id)
@@ -123,6 +119,9 @@ class HitboxIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/video',
+            video_id)
         metadata['formats'] = formats
 
         return metadata

From 14f41bc2fb01266243e763ddc80840b895acd294 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:05:08 +0600
Subject: [PATCH 0344/1515] [hitbox:live] Extract formats before metadata

---
 youtube_dl/extractor/hitbox.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index c51bc7ca4..2c440898e 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -150,10 +150,6 @@ class HitboxLiveIE(HitboxIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/live',
-            video_id)
-
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
             video_id)
@@ -194,9 +190,13 @@ class HitboxLiveIE(HitboxIE):
                             'page_url': url,
                             'player_url': 'http://www.hitbox.tv/static/player/flowplayer/flowplayer.commercial-3.2.16.swf',
                         })
-
         self._sort_formats(formats)
+
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/live',
+            video_id)
         metadata['formats'] = formats
         metadata['is_live'] = True
         metadata['title'] = self._live_title(metadata.get('title'))
+
         return metadata

From 33b066bda0491bca54cc09d7d117867f885aa5f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:09:21 +0600
Subject: [PATCH 0345/1515] [hitbox] Clarify download messages

---
 youtube_dl/extractor/hitbox.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 2c440898e..421f55bbe 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -43,7 +43,8 @@ class HitboxIE(InfoExtractor):
     def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id)
+            '%s/%s' % (url, video_id), video_id,
+            'Downloading metadata JSON')
 
         date = 'media_live_since'
         media_type = 'livestream'
@@ -90,7 +91,7 @@ class HitboxIE(InfoExtractor):
 
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
-            video_id)
+            video_id, 'Downloading video JSON')
 
         formats = []
         for video in player_config['clip']['bitrates']:

From f8e51f60b302357b43ef15cf479da3ce09643f9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Apr 2015 19:24:14 +0200
Subject: [PATCH 0346/1515] [flickr] Fix extraction (fixes #5501)

---
 youtube_dl/extractor/flickr.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 0c858b654..adffe4857 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     unescapeHTML,
@@ -29,9 +30,14 @@ class FlickrIE(InfoExtractor):
         video_id = mobj.group('id')
         video_uploader_id = mobj.group('uploader_id')
         webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
-        webpage = self._download_webpage(webpage_url, video_id)
+        req = compat_urllib_request.Request(webpage_url)
+        req.add_header(
+            'User-Agent',
+            # it needs a more recent version
+            'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20150101 Firefox/38.0 (Chrome)')
+        webpage = self._download_webpage(req, video_id)
 
-        secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, 'secret')
+        secret = self._search_regex(r'secret"\s*:\s*"(\w+)"', webpage, 'secret')
 
         first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
         first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')

From c04c3e334cc7ff0bbd2cbb8167f5cd2794c29d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Apr 2015 19:58:39 +0200
Subject: [PATCH 0347/1515] [flickr] Don't use regex for extracting the info
 from the xml files

---
 youtube_dl/extractor/flickr.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index adffe4857..2fe76d661 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
-    unescapeHTML,
+    find_xpath_attr,
 )
 
 
@@ -40,20 +40,21 @@ class FlickrIE(InfoExtractor):
         secret = self._search_regex(r'secret"\s*:\s*"(\w+)"', webpage, 'secret')
 
         first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
-        first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')
+        first_xml = self._download_xml(first_url, video_id, 'Downloading first data webpage')
 
-        node_id = self._html_search_regex(r'<Item id="id">(\d+-\d+)</Item>',
-                                          first_xml, 'node_id')
+        node_id = find_xpath_attr(
+            first_xml, './/{http://video.yahoo.com/YEP/1.0/}Item', 'id',
+            'id').text
 
         second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
-        second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage')
+        second_xml = self._download_xml(second_url, video_id, 'Downloading second data webpage')
 
         self.report_extraction(video_id)
 
-        mobj = re.search(r'<STREAM APP="(.+?)" FULLPATH="(.+?)"', second_xml)
-        if mobj is None:
+        stream = second_xml.find('.//STREAM')
+        if stream is None:
             raise ExtractorError('Unable to extract video url')
-        video_url = mobj.group(1) + unescapeHTML(mobj.group(2))
+        video_url = stream.attrib['APP'] + stream.attrib['FULLPATH']
 
         return {
             'id': video_id,

From 7513f298b04eb11ab6c0bd0845b7a3b221bee317 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Wed, 22 Apr 2015 23:50:11 +0200
Subject: [PATCH 0348/1515] [vimeo] Fix login token (fixes #5082)

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 28bcc89cd..55c1574bb 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -38,7 +38,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         login_url = 'https://vimeo.com/log_in'
         webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
         data = urlencode_postdata({
             'email': username,
             'password': password,
@@ -177,7 +177,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
         data = urlencode_postdata({
             'password': password,
             'token': token,

From 14a2d6789f24ce6f6565747a48ffe7728d5a2251 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Wed, 22 Apr 2015 23:55:19 +0200
Subject: [PATCH 0349/1515] [vimeo] one token overlooked

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 55c1574bb..9027f9dd6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -439,7 +439,7 @@ class VimeoChannelIE(InfoExtractor):
             name="([^"]+)"\s+
             value="([^"]*)"
             ''', login_form))
-        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)

From 8c8826176d4562dce9558d121a955ed71509315a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 13:49:06 +0800
Subject: [PATCH 0350/1515] [xattr] Add version detection for python-pyxattr

For more information, see #5498 and changes to convertObj() in
iustin/pyxattr@cc84e466f63906d32ec1bf4a4fcae6a7bce9a4c8
---
 youtube_dl/postprocessor/xattrpp.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 0cba99fc3..b74adff43 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     check_executable,
     hyphenate_date,
+    version_tuple,
 )
 
 
@@ -36,6 +37,19 @@ class XAttrMetadataPP(PostProcessor):
             # try the pyxattr module...
             import xattr
 
+            # Unicode arguments are not supported in python-pyxattr until
+            # version 0.5.0
+            # See https://github.com/rg3/youtube-dl/issues/5498
+            pyxattr_required_version = '0.5.0'
+            if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
+                self._downloader.report_warning(
+                    'python-pyxattr is detected but is too old. '
+                    'yourube-dl requires %s or above while your version is %s. '
+                    'Falling back to other xattr implementations' % (
+                        pyxattr_required_version, xattr.__version__))
+
+                raise ImportError
+
             def write_xattr(path, key, value):
                 return xattr.setxattr(path, key, value)
 

From 3eec9fef302929438c321adfc8ec16adcc6ffb6d Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Thu, 23 Apr 2015 11:41:21 +0200
Subject: [PATCH 0351/1515] [realvid] Add extractor for realvid.net (closes
 #5504)

---
 youtube_dl/extractor/gorillavid.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index ae24aff84..ed2623456 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -15,10 +15,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in and fastvideo.in'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -61,6 +61,15 @@ class GorillaVidIE(InfoExtractor):
             'title': 'Man of Steel - Trailer',
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        'url': 'http://realvid.net/ctn2y6p2eviw',
+        'md5': 'b2166d2cf192efd6b6d764c18fd3710e',
+        'info_dict': {
+            'id': 'ctn2y6p2eviw',
+            'ext': 'flv',
+            'title': 'rdx 1955',
+            'thumbnail': 're:http://.*\.jpg',
+        },
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,
@@ -97,7 +106,7 @@ class GorillaVidIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = self._search_regex(
-            r'style="z-index: [0-9]+;">([^<]+)</span>',
+            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'>Watch (.+) '],
             webpage, 'title', default=None) or self._og_search_title(webpage)
         video_url = self._search_regex(
             r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')

From f8610ba1ca4705dcae5c45c5ac66a99b174ebed3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 17:57:49 +0800
Subject: [PATCH 0352/1515] [ustream] Fix extraction (closes #3998)

---
 youtube_dl/extractor/ustream.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 68d03b999..d559e228a 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -39,7 +40,15 @@ class UstreamIE(InfoExtractor):
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        video_url = 'http://tcdn.ustream.tv/video/%s' % video_id
+        params = self._download_json(
+            'http://cdngw.ustream.tv/rgwjson/Viewer.getVideo/' + json.dumps({
+                'brandId': 1,
+                'videoId': int(video_id),
+                'autoplay': False,
+            }), video_id)
+
+        video_url = params['flv']
+
         webpage = self._download_webpage(url, video_id)
 
         self.report_extraction(video_id)

From 762155cc90754fc29c68bf0f0256988f37862c3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 18:07:09 +0800
Subject: [PATCH 0353/1515] [ustream] Checking errors

---
 youtube_dl/extractor/ustream.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index d559e228a..f8893b6cd 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
 )
+from ..utils import ExtractorError
 
 
 class UstreamIE(InfoExtractor):
@@ -47,6 +48,9 @@ class UstreamIE(InfoExtractor):
                 'autoplay': False,
             }), video_id)
 
+        if 'error' in params:
+            raise ExtractorError(params['error']['message'], expected=True)
+
         video_url = params['flv']
 
         webpage = self._download_webpage(url, video_id)

From 2a8137272d7d463648ccf53d25d78660209ad928 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 18:24:44 +0800
Subject: [PATCH 0354/1515] [ustream] Add an alternative approach to extract
 title (fixes #5128)

---
 youtube_dl/extractor/ustream.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index f8893b6cd..a395dc5da 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -58,7 +58,16 @@ class UstreamIE(InfoExtractor):
         self.report_extraction(video_id)
 
         video_title = self._html_search_regex(r'data-title="(?P<title>.+)"',
-                                              webpage, 'title')
+                                              webpage, 'title', default=None)
+
+        if not video_title:
+            try:
+                video_title = params['moduleConfig']['meta']['title']
+            except KeyError:
+                pass
+
+        if not video_title:
+            video_title = 'Ustream video ' + video_id
 
         uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
                                            webpage, 'uploader', fatal=False, flags=re.DOTALL)

From cd9fdccde04799910fc3ec36fb1e8baa26a5b3e0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 18:33:25 +0800
Subject: [PATCH 0355/1515] [ustream] Try to extract uploader from JSON data
 (#5128)

---
 youtube_dl/extractor/ustream.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index a395dc5da..c39c278ab 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -13,7 +13,7 @@ from ..utils import ExtractorError
 class UstreamIE(InfoExtractor):
     _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<videoID>\d+)'
     IE_NAME = 'ustream'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.ustream.tv/recorded/20274954',
         'md5': '088f151799e8f572f84eb62f17d73e5c',
         'info_dict': {
@@ -22,7 +22,18 @@ class UstreamIE(InfoExtractor):
             'uploader': 'Young Americans for Liberty',
             'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
         },
-    }
+    }, {
+        # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
+        # Title and uploader available only from params JSON
+        'url': 'http://www.ustream.tv/embed/recorded/59307601?ub=ff0000&lc=ff0000&oc=ffffff&uc=ffffff&v=3&wmode=direct',
+        'md5': '5a2abf40babeac9812ed20ae12d34e10',
+        'info_dict': {
+            'id': '59307601',
+            'ext': 'flv',
+            'title': '-CG11- Canada Games Figure Skating',
+            'uploader': 'sportscanadatv',
+        }
+    }]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
@@ -70,7 +81,13 @@ class UstreamIE(InfoExtractor):
             video_title = 'Ustream video ' + video_id
 
         uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
-                                           webpage, 'uploader', fatal=False, flags=re.DOTALL)
+                                           webpage, 'uploader', fatal=False, flags=re.DOTALL, default=None)
+
+        if not uploader:
+            try:
+                uploader = params['moduleConfig']['meta']['userName']
+            except KeyError:
+                uploader = None
 
         thumbnail = self._html_search_regex(r'<link rel="image_src" href="(?P<thumb>.*?)"',
                                             webpage, 'thumbnail', fatal=False)

From b46ed49996669a5e602042ae4d357f2ad952af58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:44:51 +0600
Subject: [PATCH 0356/1515] [cracked] Fix extraction

---
 youtube_dl/extractor/cracked.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index cf763ee7e..e579863df 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -33,17 +33,25 @@ class CrackedIE(InfoExtractor):
         video_url = self._html_search_regex(
             [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'], webpage, 'video URL')
 
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
+        title = self._search_regex(
+            [r'property="?og:title"?\s+content="([^"]+)"', r'class="?title"?>([^<]+)'],
+            webpage, 'title')
 
-        timestamp = self._html_search_regex(r'<time datetime="([^"]+)"', webpage, 'upload date', fatal=False)
+        description = self._search_regex(
+            r'name="?(?:og:)?description"?\s+content="([^"]+)"',
+            webpage, 'description', default=None)
+
+        timestamp = self._html_search_regex(
+            r'"date"\s*:\s*"([^"]+)"', webpage, 'upload date', fatal=False)
         if timestamp:
             timestamp = parse_iso8601(timestamp[:-6])
 
         view_count = str_to_int(self._html_search_regex(
-            r'<span class="views" id="viewCounts">([\d,\.]+) Views</span>', webpage, 'view count', fatal=False))
+            r'<span\s+class="?views"? id="?viewCounts"?>([\d,\.]+) Views</span>',
+            webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'<span id="commentCounts">([\d,\.]+)</span>', webpage, 'comment count', fatal=False))
+            r'<span\s+id="?commentCounts"?>([\d,\.]+)</span>',
+            webpage, 'comment count', fatal=False))
 
         m = re.search(r'_(?P<width>\d+)X(?P<height>\d+)\.mp4$', video_url)
         if m:

From 6447353f52dc4b2b5926d3d9a3a28d3a7a242b4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:49:54 +0600
Subject: [PATCH 0357/1515] [cracked] Add support for youtube embeds

---
 youtube_dl/extractor/cracked.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index e579863df..8b381dd3a 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -30,8 +30,15 @@ class CrackedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        youtube_url = self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
+            webpage, 'youtube url', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url)
+
         video_url = self._html_search_regex(
-            [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'], webpage, 'video URL')
+            [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'],
+            webpage, 'video URL')
 
         title = self._search_regex(
             [r'property="?og:title"?\s+content="([^"]+)"', r'class="?title"?>([^<]+)'],

From c610f38ba9586f21f632e115cbc15e172d60ee40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:58:50 +0600
Subject: [PATCH 0358/1515] [cracked] Update tests

---
 youtube_dl/extractor/cracked.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index 8b381dd3a..b2b151558 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -11,18 +11,30 @@ from ..utils import (
 
 class CrackedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cracked\.com/video_(?P<id>\d+)_[\da-z-]+\.html'
-    _TEST = {
-        'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html',
-        'md5': '4b29a5eeec292cd5eca6388c7558db9e',
+    _TESTS = [{
+        'url': 'http://www.cracked.com/video_19070_if-animal-actors-got-e21-true-hollywood-stories.html',
+        'md5': '89b90b9824e3806ca95072c4d78f13f7',
         'info_dict': {
-            'id': '19006',
+            'id': '19070',
             'ext': 'mp4',
-            'title': '4 Plot Holes You Didn\'t Notice in Your Favorite Movies',
-            'description': 'md5:3b909e752661db86007d10e5ec2df769',
-            'timestamp': 1405659600,
-            'upload_date': '20140718',
+            'title': 'If Animal Actors Got E! True Hollywood Stories',
+            'timestamp': 1404954000,
+            'upload_date': '20140710',
         }
-    }
+    }, {
+        # youtube embed
+        'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html',
+        'md5': 'ccd52866b50bde63a6ef3b35016ba8c7',
+        'info_dict': {
+            'id': 'EjI00A3rZD0',
+            'ext': 'mp4',
+            'title': "4 Plot Holes You Didn't Notice in Your Favorite Movies - The Spit Take",
+            'description': 'md5:c603708c718b796fe6079e2b3351ffc7',
+            'upload_date': '20140725',
+            'uploader_id': 'Cracked',
+            'uploader': 'Cracked',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -34,7 +46,7 @@ class CrackedIE(InfoExtractor):
             r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
             webpage, 'youtube url', default=None)
         if youtube_url:
-            return self.url_result(youtube_url)
+            return self.url_result(youtube_url, 'Youtube')
 
         video_url = self._html_search_regex(
             [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'],

From d7403332248a3893810f0461a682229552e1fd9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:59:18 +0600
Subject: [PATCH 0359/1515] [cracked] Modernize

---
 youtube_dl/extractor/cracked.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index b2b151558..94d03ce2a 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -37,8 +37,7 @@ class CrackedIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 

From 4515cb43cadd583de40b38021202595248080fb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 22:11:09 +0600
Subject: [PATCH 0360/1515] [xattrpp] Fix typo

---
 youtube_dl/postprocessor/xattrpp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index b74adff43..93d0abcf6 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -44,7 +44,7 @@ class XAttrMetadataPP(PostProcessor):
             if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
                 self._downloader.report_warning(
                     'python-pyxattr is detected but is too old. '
-                    'yourube-dl requires %s or above while your version is %s. '
+                    'youtube-dl requires %s or above while your version is %s. '
                     'Falling back to other xattr implementations' % (
                         pyxattr_required_version, xattr.__version__))
 

From 02f502f435f01d82b71d52e8b5fdb9369d76873d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 01:34:57 +0600
Subject: [PATCH 0361/1515] [README] Document on how to enable old format
 selection behavior (#5510, #5511)

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7dbe66995..3ea69b637 100644
--- a/README.md
+++ b/README.md
@@ -270,7 +270,9 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-youtube-dl uses `-f bestvideo+bestaudio/best` if ffmpeg or avconv are installed (`best` is needed for videos that don't come from YouTube because they don't provide the audio and video in two different files). If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+
+If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl/blob/master/README.md#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From 7fb993e1f4105d573a5e42f71a2b3c51c0794a9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 01:38:02 +0600
Subject: [PATCH 0362/1515] [README] Fix configuration file link and typo

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3ea69b637..2b3bdde22 100644
--- a/README.md
+++ b/README.md
@@ -268,11 +268,11 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
 Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
 
-If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl/blob/master/README.md#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From 4d6a3ff411d494e2f3168bca0331317718b5d9a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 01:41:36 +0600
Subject: [PATCH 0363/1515] [README] Finally fix configuration file link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2b3bdde22..7040be1ec 100644
--- a/README.md
+++ b/README.md
@@ -272,7 +272,7 @@ If you want to download multiple videos and they don't have the same formats ava
 
 Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
 
-If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From a9b0d4e1f4936d93f277f098495eb4d5b770056f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 14:09:35 +0800
Subject: [PATCH 0364/1515] [Crunchyroll] Fix extraction on Python 2.6

XPath with recursive children selection not supported
---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 6ded723c9..1c77df47e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -263,8 +263,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             streamdata = self._download_xml(
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
-            video_url = streamdata.find('.//host').text
-            video_play_path = streamdata.find('.//file').text
+            video_url = streamdata.find('./host').text
+            video_play_path = streamdata.find('./file').text
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From ddbed36455f8b3053f38d84b2e62e2fb5cd66eac Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 08:48:49 +0200
Subject: [PATCH 0365/1515] [embedthumbnail] Add support for mp3 cover
 embedding

---
 youtube_dl/__init__.py                     |  4 +-
 youtube_dl/postprocessor/__init__.py       |  4 +-
 youtube_dl/postprocessor/atomicparsley.py  | 62 ---------------
 youtube_dl/postprocessor/embedthumbnail.py | 87 ++++++++++++++++++++++
 youtube_dl/postprocessor/ffmpeg.py         |  2 +
 5 files changed, 92 insertions(+), 67 deletions(-)
 delete mode 100644 youtube_dl/postprocessor/atomicparsley.py
 create mode 100644 youtube_dl/postprocessor/embedthumbnail.py

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 1c8b411b7..d7759db68 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -240,9 +240,7 @@ def _real_main(argv=None):
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
     if opts.embedthumbnail:
-        if not opts.addmetadata:
-            postprocessors.append({'key': 'FFmpegAudioFix'})
-        postprocessors.append({'key': 'AtomicParsley'})
+        postprocessors.append({'key': 'EmbedThumbnail'})
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index f39acadce..ab7f1a29a 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from .atomicparsley import AtomicParsleyPP
+from .embedthumbnail import EmbedThumbnailPP
 from .ffmpeg import (
     FFmpegPostProcessor,
     FFmpegAudioFixPP,
@@ -23,7 +23,7 @@ def get_postprocessor(key):
 
 
 __all__ = [
-    'AtomicParsleyPP',
+    'EmbedThumbnailPP',
     'ExecAfterDownloadPP',
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
deleted file mode 100644
index e4e198695..000000000
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-
-import os
-import subprocess
-
-from .common import PostProcessor
-from ..compat import (
-    compat_urlretrieve,
-)
-from ..utils import (
-    check_executable,
-    encodeFilename,
-    PostProcessingError,
-    prepend_extension,
-    shell_quote
-)
-
-
-class AtomicParsleyPPError(PostProcessingError):
-    pass
-
-
-class AtomicParsleyPP(PostProcessor):
-    def run(self, info):
-        if not check_executable('AtomicParsley', ['-v']):
-            raise AtomicParsleyPPError('AtomicParsley was not found. Please install.')
-
-        filename = info['filepath']
-        temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = prepend_extension(filename, 'thumb')
-
-        if not info.get('thumbnail'):
-            raise AtomicParsleyPPError('Thumbnail was not found. Nothing to do.')
-
-        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
-
-        cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
-
-        self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
-
-        if self._downloader.params.get('verbose', False):
-            self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
-
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
-
-        if p.returncode != 0:
-            msg = stderr.decode('utf-8', 'replace').strip()
-            raise AtomicParsleyPPError(msg)
-
-        os.remove(encodeFilename(temp_thumbnail))
-        # for formats that don't support thumbnails (like 3gp) AtomicParsley
-        # won't create to the temporary file
-        if b'No changes' in stdout:
-            self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
-        else:
-            os.remove(encodeFilename(filename))
-            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
-
-        return [], info
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
new file mode 100644
index 000000000..b6507db27
--- /dev/null
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+
+import os
+import subprocess
+
+from .common import PostProcessor
+from ..compat import (
+    compat_urlretrieve,
+)
+from ..utils import (
+    check_executable,
+    encodeFilename,
+    PostProcessingError,
+    prepend_extension,
+    shell_quote
+)
+
+
+class EmbedThumbnailPPError(PostProcessingError):
+    pass
+
+
+class EmbedThumbnailPP(PostProcessor):
+    def run(self, info):
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+        temp_thumbnail = prepend_extension(filename, 'thumb')
+
+        if not info.get('thumbnail'):
+            raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')
+
+        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
+
+        if info['ext'] == 'mp3':
+            if not check_executable('ffmpeg', ['-version']):
+                raise AtomicParsleyPPError('FFmpeg was not found. Please install.')
+
+            cmd = ['ffmpeg', '-i', filename, '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1', '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"', temp_filename]
+
+            self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
+
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] FFmpeg command line: %s' % shell_quote(cmd))
+
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = p.communicate()
+
+            if p.returncode != 0:
+                msg = stderr.decode('utf-8', 'replace').strip()
+                raise EmbedThumbnailPPError(msg)
+
+            os.remove(encodeFilename(temp_thumbnail))
+            os.remove(encodeFilename(filename))
+            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        elif info['ext'] == 'm4a':
+            if not check_executable('AtomicParsley', ['-v']):
+                raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
+
+            cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
+
+            self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
+
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
+
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = p.communicate()
+
+            if p.returncode != 0:
+                msg = stderr.decode('utf-8', 'replace').strip()
+                raise EmbedThumbnailPPError(msg)
+
+            os.remove(encodeFilename(temp_thumbnail))
+            # for formats that don't support thumbnails (like 3gp) AtomicParsley
+            # won't create to the temporary file
+            if b'No changes' in stdout:
+                self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
+            else:
+                os.remove(encodeFilename(filename))
+                os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+        else:
+            raise EmbedThumbnailPPError('Only mp3 and m4a are supported for thumbnail embedding for now.')
+
+        return [], info
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index df6fb6665..c01f2eedd 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -280,6 +280,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
+        information['ext'] = extension
+
         return [path], information
 
 

From 31fd9c76013faaa86b0c515d9305de548856ef84 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 09:08:57 +0200
Subject: [PATCH 0366/1515] [embedthumbnail] use FFmpegPostProcessor for mp3

---
 youtube_dl/postprocessor/embedthumbnail.py | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index b6507db27..a2d6b14db 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 import os
 import subprocess
 
-from .common import PostProcessor
+from .ffmpeg import FFmpegPostProcessor
+
 from ..compat import (
     compat_urlretrieve,
 )
@@ -22,7 +23,7 @@ class EmbedThumbnailPPError(PostProcessingError):
     pass
 
 
-class EmbedThumbnailPP(PostProcessor):
+class EmbedThumbnailPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -34,22 +35,12 @@ class EmbedThumbnailPP(PostProcessor):
         compat_urlretrieve(info['thumbnail'], temp_thumbnail)
 
         if info['ext'] == 'mp3':
-            if not check_executable('ffmpeg', ['-version']):
-                raise AtomicParsleyPPError('FFmpeg was not found. Please install.')
-
-            cmd = ['ffmpeg', '-i', filename, '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1', '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"', temp_filename]
+            options = ['-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+                '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
-            if self._downloader.params.get('verbose', False):
-                self._downloader.to_screen('[debug] FFmpeg command line: %s' % shell_quote(cmd))
-
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            stdout, stderr = p.communicate()
-
-            if p.returncode != 0:
-                msg = stderr.decode('utf-8', 'replace').strip()
-                raise EmbedThumbnailPPError(msg)
+            self.run_ffmpeg(filename, temp_filename, options)
 
             os.remove(encodeFilename(temp_thumbnail))
             os.remove(encodeFilename(filename))

From c0ea8ebb9bc628e3787e739dbf58bd0e59c83f87 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 09:11:39 +0200
Subject: [PATCH 0367/1515] [ffmpeg] Remove unneeded class

---
 youtube_dl/postprocessor/ffmpeg.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index c01f2eedd..7a952963e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -590,21 +590,6 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         return info['__files_to_merge'], info
 
 
-class FFmpegAudioFixPP(FFmpegPostProcessor):
-    def run(self, info):
-        filename = info['filepath']
-        temp_filename = prepend_extension(filename, 'temp')
-
-        options = ['-vn', '-acodec', 'copy']
-        self._downloader.to_screen('[ffmpeg] Fixing audio file "%s"' % filename)
-        self.run_ffmpeg(filename, temp_filename, options)
-
-        os.remove(encodeFilename(filename))
-        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
-
-        return [], info
-
-
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')

From 10fb7710e85a240cc4bd065b98fd3f5a0c3d10ca Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 09:17:46 +0200
Subject: [PATCH 0368/1515] Forgot to clean the remains of class

---
 youtube_dl/postprocessor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index ab7f1a29a..0d8ef6ca2 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 from .embedthumbnail import EmbedThumbnailPP
 from .ffmpeg import (
     FFmpegPostProcessor,
-    FFmpegAudioFixPP,
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
@@ -25,7 +24,6 @@ def get_postprocessor(key):
 __all__ = [
     'EmbedThumbnailPP',
     'ExecAfterDownloadPP',
-    'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
     'FFmpegFixupM4aPP',

From 25f7d1beba61ad9a5d013d423f169eafb05d76a5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 22:25:34 +0800
Subject: [PATCH 0369/1515] [gdcvault] Extend _VALID_URL (fixes #5236)

---
 youtube_dl/extractor/gdcvault.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 51796f3a4..f634e6285 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -11,7 +11,7 @@ from ..utils import remove_end
 
 
 class GDCVaultIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)'
+    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)?'
     _NETRC_MACHINE = 'gdcvault'
     _TESTS = [
         {
@@ -43,6 +43,10 @@ class GDCVaultIE(InfoExtractor):
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
             },
             'skip': 'Requires login',
+        },
+        {
+            'url': 'http://gdcvault.com/play/1020791/',
+            'only_matching': True,
         }
     ]
 

From c8ff645766aa56742045adcf0c64b92617334eb5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 22:43:33 +0800
Subject: [PATCH 0370/1515] [gdcvault] Add display_id

---
 youtube_dl/extractor/gdcvault.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index f634e6285..43f916412 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -19,6 +19,7 @@ class GDCVaultIE(InfoExtractor):
             'md5': '7ce8388f544c88b7ac11c7ab1b593704',
             'info_dict': {
                 'id': '1019721',
+                'display_id': 'Doki-Doki-Universe-Sweet-Simple',
                 'ext': 'mp4',
                 'title': 'Doki-Doki Universe: Sweet, Simple and Genuine (GDC Next 10)'
             }
@@ -27,6 +28,7 @@ class GDCVaultIE(InfoExtractor):
             'url': 'http://www.gdcvault.com/play/1015683/Embracing-the-Dark-Art-of',
             'info_dict': {
                 'id': '1015683',
+                'display_id': 'Embracing-the-Dark-Art-of',
                 'ext': 'flv',
                 'title': 'Embracing the Dark Art of Mathematical Modeling in AI'
             },
@@ -39,6 +41,7 @@ class GDCVaultIE(InfoExtractor):
             'md5': 'a5eb77996ef82118afbbe8e48731b98e',
             'info_dict': {
                 'id': '1015301',
+                'display_id': 'Thexder-Meets-Windows-95-or',
                 'ext': 'flv',
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
             },
@@ -94,7 +97,7 @@ class GDCVaultIE(InfoExtractor):
         })
         return video_formats
 
-    def _login(self, webpage_url, video_id):
+    def _login(self, webpage_url, display_id):
         (username, password) = self._get_login_info()
         if username is None or password is None:
             self.report_warning('It looks like ' + webpage_url + ' requires a login. Try specifying a username and password and try again.')
@@ -111,9 +114,9 @@ class GDCVaultIE(InfoExtractor):
 
         request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        self._download_webpage(request, video_id, 'Logging in')
-        start_page = self._download_webpage(webpage_url, video_id, 'Getting authenticated video page')
-        self._download_webpage(logout_url, video_id, 'Logging out')
+        self._download_webpage(request, display_id, 'Logging in')
+        start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
+        self._download_webpage(logout_url, display_id, 'Logging out')
 
         return start_page
 
@@ -121,8 +124,10 @@ class GDCVaultIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         video_id = mobj.group('id')
+        display_id = mobj.group('name') or video_id
+
         webpage_url = 'http://www.gdcvault.com/play/' + video_id
-        start_page = self._download_webpage(webpage_url, video_id)
+        start_page = self._download_webpage(webpage_url, display_id)
 
         direct_url = self._search_regex(
             r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
@@ -135,6 +140,7 @@ class GDCVaultIE(InfoExtractor):
 
             return {
                 'id': video_id,
+                'display_id': display_id,
                 'url': video_url,
                 'ext': 'flv',
                 'title': title,
@@ -145,7 +151,7 @@ class GDCVaultIE(InfoExtractor):
             start_page, 'xml root', default=None)
         if xml_root is None:
             # Probably need to authenticate
-            login_res = self._login(webpage_url, video_id)
+            login_res = self._login(webpage_url, display_id)
             if login_res is None:
                 self.report_warning('Could not login.')
             else:
@@ -163,7 +169,7 @@ class GDCVaultIE(InfoExtractor):
             xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
 
         xml_decription_url = xml_root + 'xml/' + xml_name
-        xml_description = self._download_xml(xml_decription_url, video_id)
+        xml_description = self._download_xml(xml_decription_url, display_id)
 
         video_title = xml_description.find('./metadata/title').text
         video_formats = self._parse_mp4(xml_description)
@@ -172,6 +178,7 @@ class GDCVaultIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': video_title,
             'formats': video_formats,
         }

From 5090d93f2c7e5d40cd6d7a8c9eda789f67bd1eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 21:47:13 +0600
Subject: [PATCH 0371/1515] [dotsub] Fix extraction

---
 youtube_dl/extractor/dotsub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index f51d88a98..e9ca236d4 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -36,7 +36,8 @@ class DotsubIE(InfoExtractor):
         if not video_url:
             webpage = self._download_webpage(url, video_id)
             video_url = self._search_regex(
-                r'"file"\s*:\s*\'([^\']+)', webpage, 'video url')
+                [r'<source[^>]+src="([^"]+)"', r'"file"\s*:\s*\'([^\']+)'],
+                webpage, 'video url')
 
         return {
             'id': video_id,

From 2ad978532bd0a94fb7529f429a26e9b9966b2e1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:03:14 +0600
Subject: [PATCH 0372/1515] [ellentv] Fix extraction

---
 youtube_dl/extractor/ellentv.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 5154bbd7f..93affaa8f 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -39,24 +39,20 @@ class EllenTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://widgets.ellentube.com/videos/%s' % video_id,
+            video_id)
 
-        video_url = self._html_search_meta('VideoURL', webpage, 'url', fatal=True)
-        title = self._og_search_title(webpage, default=None) or self._search_regex(
-            r'pageName\s*=\s*"([^"]+)"', webpage, 'title')
-        description = self._html_search_meta(
-            'description', webpage, 'description') or self._og_search_description(webpage)
-        timestamp = parse_iso8601(self._search_regex(
-            r'<span class="publish-date"><time datetime="([^"]+)">',
-            webpage, 'timestamp', fatal=False))
+        partner_id = self._search_regex(
+            r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id')
 
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-        }
+        kaltura_id = self._search_regex(
+            [r'id="kaltura_player_([^"]+)"',
+             r"_wb_entry_id\s*:\s*'([^']+)",
+             r'data-kaltura-entry-id="([^"]+)'],
+            webpage, 'kaltura id')
+
+        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')
 
 
 class EllenTVClipsIE(InfoExtractor):

From 870744ce8f03bdd50a1d4d4ba87913aecba35716 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:07:15 +0600
Subject: [PATCH 0373/1515] [ellentv] Fix tests

---
 youtube_dl/extractor/ellentv.py | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 93affaa8f..464aeabdb 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -12,29 +12,19 @@ from ..utils import (
 
 class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
         'md5': '8e3c576bf2e9bfff4d76565f56f94c9c',
         'info_dict': {
-            'id': '0-ipq1gsai',
+            'id': '0_ipq1gsai',
             'ext': 'mp4',
             'title': 'Fast Fingers of Fate',
-            'description': 'md5:686114ced0a032926935e9015ee794ac',
-            'timestamp': 1428033600,
+            'description': 'md5:587e79fbbd0d73b148bc596d99ce48e6',
+            'timestamp': 1428035648,
             'upload_date': '20150403',
+            'uploader_id': 'batchUser',
         }
-    }, {
-        'url': 'http://ellentube.com/videos/0-dvzmabd5/',
-        'md5': '98238118eaa2bbdf6ad7f708e3e4f4eb',
-        'info_dict': {
-            'id': '0-dvzmabd5',
-            'ext': 'mp4',
-            'title': '1 year old twin sister makes her brother laugh',
-            'description': '1 year old twin sister makes her brother laugh',
-            'timestamp': 1419542075,
-            'upload_date': '20141225',
-        }
-    }]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 66be4b89d73adcfeb9cd31cd0c903ecca40e2152 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:09:54 +0600
Subject: [PATCH 0374/1515] [ellentv:clips] Fix extraction

---
 youtube_dl/extractor/ellentv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 464aeabdb..54601786a 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -78,4 +78,8 @@ class EllenTVClipsIE(InfoExtractor):
             raise ExtractorError('Failed to download JSON', cause=ve)
 
     def _extract_entries(self, playlist):
-        return [self.url_result(item['url'], 'EllenTV') for item in playlist]
+        return [
+            self.url_result(
+                'kaltura:%s:%s' % (item['kaltura_partner_id'], item['kaltura_entry_id']),
+                'Kaltura')
+            for item in playlist]

From d0aefec99aa7bf2cd307b700f7c0d2c268d6762d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:10:27 +0600
Subject: [PATCH 0375/1515] [ellentv:clips] Fix test

---
 youtube_dl/extractor/ellentv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 54601786a..74b50bca2 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -54,7 +54,7 @@ class EllenTVClipsIE(InfoExtractor):
             'id': 'meryl-streep-vanessa-hudgens',
             'title': 'Meryl Streep, Vanessa Hudgens',
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 7,
     }
 
     def _real_extract(self, url):

From db37e0c273d9d139d1d6a8541146d929b659610d Mon Sep 17 00:00:00 2001
From: mrkrossxdx <mrkrossxdx@libero.it>
Date: Fri, 24 Apr 2015 20:50:34 +0200
Subject: [PATCH 0376/1515] Added support for mpv if mplayer is not available

---
 youtube_dl/downloader/mplayer.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 72cef30ea..551c4ae94 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -16,12 +16,19 @@ class MplayerFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = [
-            'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
-            '-dumpstream', '-dumpfile', tmpfilename, url]
+        args = []
         # Check for mplayer first
-        if not check_executable('mplayer', ['-h']):
-            self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
+        if check_executable('mplayer', ['-h']):
+            args = [
+                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+                '-dumpstream', '-dumpfile', tmpfilename, url]
+                        
+        # Check for mpv
+        elif check_executable('mpv', ['-h']):
+            args = [
+                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
+        else:
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run')
             return False
 
         # Download using mplayer.

From a4196c3ea57cf06b6d1f961b9c40f65e173c9876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 24 Apr 2015 22:06:22 +0200
Subject: [PATCH 0377/1515] [ellentv] Remove unused import

---
 youtube_dl/extractor/ellentv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 74b50bca2..02c6a4615 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -6,7 +6,6 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_iso8601,
 )
 
 

From 92995e6265d2287ee67af869e36a56a05fb86a06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 24 Apr 2015 22:08:00 +0200
Subject: [PATCH 0378/1515] [postprocessor/embedthumbnail] Style fix

---
 youtube_dl/postprocessor/embedthumbnail.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index a2d6b14db..7ba98a0ea 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -35,7 +35,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
         compat_urlretrieve(info['thumbnail'], temp_thumbnail)
 
         if info['ext'] == 'mp3':
-            options = ['-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+            options = [
+                '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)

From 9f3fa89f7c6300c8a7372cf7f694fd8a659785cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Apr 2015 11:59:54 +0200
Subject: [PATCH 0379/1515] Remove the --max-quality option

It doesn't work well with 'bestvideo' and 'bestaudio' because they are usually before the max quality.
Format filters should be used instead, they are more flexible and don't require the requested quality to exist for each video.
---
 test/parameters.json    |  1 -
 test/test_YoutubeDL.py  | 33 ---------------------------------
 youtube_dl/YoutubeDL.py |  8 --------
 youtube_dl/__init__.py  |  1 -
 youtube_dl/options.py   |  4 ----
 youtube_dl/utils.py     |  9 ---------
 6 files changed, 56 deletions(-)

diff --git a/test/parameters.json b/test/parameters.json
index 48b5a062e..7bf59c25f 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -8,7 +8,6 @@
     "forcetitle": false, 
     "forceurl": false, 
     "format": "best",
-    "format_limit": null, 
     "ignoreerrors": false, 
     "listformats": null, 
     "logtostderr": false, 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 820e55ec2..bb4a65ee1 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -101,39 +101,6 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['ext'], 'flv')
 
-    def test_format_limit(self):
-        formats = [
-            {'format_id': 'meh', 'url': 'http://example.com/meh', 'preference': 1},
-            {'format_id': 'good', 'url': 'http://example.com/good', 'preference': 2},
-            {'format_id': 'great', 'url': 'http://example.com/great', 'preference': 3},
-            {'format_id': 'excellent', 'url': 'http://example.com/exc', 'preference': 4},
-        ]
-        info_dict = _make_result(formats)
-
-        ydl = YDL()
-        ydl.process_ie_result(info_dict)
-        downloaded = ydl.downloaded_info_dicts[0]
-        self.assertEqual(downloaded['format_id'], 'excellent')
-
-        ydl = YDL({'format_limit': 'good'})
-        assert ydl.params['format_limit'] == 'good'
-        ydl.process_ie_result(info_dict.copy())
-        downloaded = ydl.downloaded_info_dicts[0]
-        self.assertEqual(downloaded['format_id'], 'good')
-
-        ydl = YDL({'format_limit': 'great', 'format': 'all'})
-        ydl.process_ie_result(info_dict.copy())
-        self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'meh')
-        self.assertEqual(ydl.downloaded_info_dicts[1]['format_id'], 'good')
-        self.assertEqual(ydl.downloaded_info_dicts[2]['format_id'], 'great')
-        self.assertTrue('3' in ydl.msgs[0])
-
-        ydl = YDL()
-        ydl.params['format_limit'] = 'excellent'
-        ydl.process_ie_result(info_dict.copy())
-        downloaded = ydl.downloaded_info_dicts[0]
-        self.assertEqual(downloaded['format_id'], 'excellent')
-
     def test_format_selection(self):
         formats = [
             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0fdcf1b0b..977141881 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -64,7 +64,6 @@ from .utils import (
     sanitize_path,
     std_headers,
     subtitles_filename,
-    takewhile_inclusive,
     UnavailableVideoError,
     url_basename,
     version_tuple,
@@ -135,7 +134,6 @@ class YoutubeDL(object):
                        (or video) as a single JSON line.
     simulate:          Do not download the video files.
     format:            Video format code. See options.py for more information.
-    format_limit:      Highest quality format to try.
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
@@ -1068,12 +1066,6 @@ class YoutubeDL(object):
             full_format_info.update(format)
             format['http_headers'] = self._calc_headers(full_format_info)
 
-        format_limit = self.params.get('format_limit', None)
-        if format_limit:
-            formats = list(takewhile_inclusive(
-                lambda f: f['format_id'] != format_limit, formats
-            ))
-
         # TODO Central sorting goes here
 
         if formats[0] is not info_dict:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index d7759db68..c88489f29 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -283,7 +283,6 @@ def _real_main(argv=None):
         'simulate': opts.simulate or any_getting,
         'skip_download': opts.skip_download,
         'format': opts.format,
-        'format_limit': opts.format_limit,
         'listformats': opts.listformats,
         'outtmpl': outtmpl,
         'autonumber_size': opts.autonumber_size,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 39c38c980..4c9d39d9a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -331,10 +331,6 @@ def parseOpts(overrideArguments=None):
         '--prefer-free-formats',
         action='store_true', dest='prefer_free_formats', default=False,
         help='Prefer free video formats unless a specific one is requested')
-    video_format.add_option(
-        '--max-quality',
-        action='store', dest='format_limit', metavar='FORMAT',
-        help='Highest quality format to download')
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index edeee1853..c69d3e165 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1109,15 +1109,6 @@ def shell_quote(args):
     return ' '.join(quoted_args)
 
 
-def takewhile_inclusive(pred, seq):
-    """ Like itertools.takewhile, but include the latest evaluated element
-        (the first element so that Not pred(e)) """
-    for e in seq:
-        yield e
-        if not pred(e):
-            return
-
-
 def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 

From 0d1bd5d62f29cc1a02c4dd5cea55fe018f9fea97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Apr 2015 15:14:16 +0200
Subject: [PATCH 0380/1515] README: remove --max-quality

---
 README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 7040be1ec..948e0a4b9 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,6 @@ which means you can modify it, redistribute it or use it however you like.
     -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
-    --max-quality FORMAT             Highest quality format to download
     -F, --list-formats               List all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
     --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
@@ -324,9 +323,9 @@ YouTube changed their playlist format in March 2014 and later on, so you'll need
 
 If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
-### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
+### Do I always have to pass `-citw`?
 
-By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, `--max-quality` *limits* the video quality (so if you want the best quality, do NOT pass it in), and the only option out of `-citw` that is regularly useful is `-i`.
+By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
 ### Can you please put the -b option back?
 

From a542e372ab3dc36d2320ba588d5bfa8011e1df5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 20:22:20 +0600
Subject: [PATCH 0381/1515] [mtv] Stuff lang into info URL when available

---
 youtube_dl/extractor/mtv.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 4430b3416..b48fac5e3 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -25,6 +25,7 @@ def _media_xml_tag(tag):
 
 class MTVServicesInfoExtractor(InfoExtractor):
     _MOBILE_TEMPLATE = None
+    _LANG = None
 
     @staticmethod
     def _id_from_uri(uri):
@@ -169,8 +170,12 @@ class MTVServicesInfoExtractor(InfoExtractor):
         video_id = self._id_from_uri(uri)
         feed_url = self._get_feed_url(uri)
         data = compat_urllib_parse.urlencode({'uri': uri})
+        info_url = feed_url + '?'
+        if self._LANG:
+            info_url += 'lang=%s&' % self._LANG
+        info_url += data
         idoc = self._download_xml(
-            feed_url + '?' + data, video_id,
+            info_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
         return self.playlist_result(
             [self._get_video_info(item) for item in idoc.findall('.//item')])

From e4a5e772f2e691a2f462796ad08dea6e04938697 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 20:23:42 +0600
Subject: [PATCH 0382/1515] [southpark:espanol] Add extractor (Closes #5525)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a64afa1da..ab80fd5e0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -478,6 +478,7 @@ from .soundgasm import (
 )
 from .southpark import (
     SouthParkIE,
+    SouthParkEsIE,
     SouthparkDeIE,
 )
 from .space import SpaceIE
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index c20397b3d..5c653c8bc 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -20,6 +20,17 @@ class SouthParkIE(MTVServicesInfoExtractor):
     }]
 
 
+class SouthParkEsIE(SouthParkIE):
+    IE_NAME = 'southpark.cc.com:espanol'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
+    _LANG = 'es'
+
+    _TESTS = [{
+        'url': 'http://southpark.cc.com/episodios-en-espanol/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
+        'playlist_count': 4,
+    }]
+
+
 class SouthparkDeIE(SouthParkIE):
     IE_NAME = 'southpark.de'
     _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'

From 857f00ed94ea04575a19155d8813b7b599bbd6a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 20:24:15 +0600
Subject: [PATCH 0383/1515] [southpark] Improve some _VALID_URL's

---
 youtube_dl/extractor/southpark.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 5c653c8bc..77758bbed 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -5,7 +5,7 @@ from .mtv import MTVServicesInfoExtractor
 
 class SouthParkIE(MTVServicesInfoExtractor):
     IE_NAME = 'southpark.cc.com'
-    _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.cc\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
 
     _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
 
@@ -33,7 +33,7 @@ class SouthParkEsIE(SouthParkIE):
 
 class SouthparkDeIE(SouthParkIE):
     IE_NAME = 'southpark.de'
-    _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.de/(?:clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'
 
     _TESTS = [{

From 529d26c3e191a3fc24b9c7618310c47929a0c11f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 21:06:27 +0600
Subject: [PATCH 0384/1515] [orf:iptv] Update test

---
 youtube_dl/extractor/orf.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index ca1a5bb3c..2e6c9872b 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -210,16 +210,16 @@ class ORFIPTVIE(InfoExtractor):
     _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://iptv.orf.at/stories/2267952',
-        'md5': '26ffa4bab6dbce1eee78bbc7021016cd',
+        'url': 'http://iptv.orf.at/stories/2275236/',
+        'md5': 'c8b22af4718a4b4af58342529453e3e5',
         'info_dict': {
-            'id': '339775',
+            'id': '350612',
             'ext': 'flv',
-            'title': 'Kreml-Kritiker Nawalny wieder frei',
-            'description': 'md5:6f24e7f546d364dacd0e616a9e409236',
-            'duration': 84.729,
+            'title': 'Weitere Evakuierungen um Vulkan Calbuco',
+            'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
+            'duration': 68.197,
             'thumbnail': 're:^https?://.*\.jpg$',
-            'upload_date': '20150306',
+            'upload_date': '20150425',
         },
     }
 

From 672f1bd8497f43179dcd01f8b4831564f0b42356 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 23:46:51 +0800
Subject: [PATCH 0385/1515] [cspan] Extract subtitles

---
 youtube_dl/extractor/cspan.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index d516b1402..fbefd37d0 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -8,6 +8,7 @@ from ..utils import (
     unescapeHTML,
     find_xpath_attr,
     smuggle_url,
+    determine_ext,
 )
 from .senateisvp import SenateISVPIE
 
@@ -87,6 +88,10 @@ class CSpanIE(InfoExtractor):
             return self.url_result(surl, 'SenateISVP', video_id, title)
 
         files = data['video']['files']
+        try:
+            capfile = data['video']['capfile']['#text']
+        except KeyError:
+            capfile = None
 
         entries = [{
             'id': '%s_%d' % (video_id, partnum + 1),
@@ -97,6 +102,12 @@ class CSpanIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': int_or_none(f.get('length', {}).get('#text')),
+            'subtitles': {
+                'en': [{
+                    'url': capfile,
+                    'ext': determine_ext(capfile, 'dfxp')
+                }],
+            } if capfile else None,
         } for partnum, f in enumerate(files)]
 
         if len(entries) == 1:

From bf6427d2fbcbd95cd1cb640e8b894c18782a2a12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Apr 2015 23:15:05 +0800
Subject: [PATCH 0386/1515] [ffmpeg] Add dfxp (TTML) subtitles support (#3432,
 #5146)

---
 test/test_utils.py                 | 38 +++++++++++++++++++++
 youtube_dl/postprocessor/ffmpeg.py | 25 ++++++++++++++
 youtube_dl/utils.py                | 53 ++++++++++++++++++++++++++++++
 3 files changed, 116 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2e3a6480c..17017a8c0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -58,6 +58,8 @@ from youtube_dl.utils import (
     xpath_text,
     render_table,
     match_str,
+    parse_dfxp_time_expr,
+    dfxp2srt,
 )
 
 
@@ -581,6 +583,42 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
+    def test_parse_dfxp_time_expr(self):
+        self.assertEqual(parse_dfxp_time_expr(None), 0.0)
+        self.assertEqual(parse_dfxp_time_expr(''), 0.0)
+        self.assertEqual(parse_dfxp_time_expr('0.1'), 0.1)
+        self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
+        self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
+        self.assertEqual(parse_dfxp_time_expr('00:00:01.100'), 1.1)
+
+    def test_dfxp2srt(self):
+        dfxp_data = '''<?xml version="1.0" encoding="UTF-8"?>
+            <tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
+                    <p begin="1" end="2">第二行<br/>♪♪</p>
+                    <p begin="2" end="3"><span>Third<br/>Line</span></p>
+                </div>
+            </body>
+            </tt>'''
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+The following line contains Chinese characters and special symbols
+
+2
+00:00:01,000 --> 00:00:02,000
+第二行
+♪♪
+
+3
+00:00:02,000 --> 00:00:03,000
+Third
+Line
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data), srt_data)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7a952963e..1765f4969 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -20,6 +20,7 @@ from ..utils import (
     prepend_extension,
     shell_quote,
     subtitles_filename,
+    dfxp2srt,
 )
 
 
@@ -651,6 +652,30 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'format' % new_ext)
                 continue
             new_file = subtitles_filename(filename, lang, new_ext)
+
+            if ext == 'dfxp' or ext == 'ttml':
+                self._downloader.report_warning(
+                    'You have requested to convert dfxp (TTML) subtitles into another format, '
+                    'which results in style information loss')
+
+                dfxp_file = subtitles_filename(filename, lang, ext)
+                srt_file = subtitles_filename(filename, lang, 'srt')
+
+                with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
+                    srt_data = dfxp2srt(f.read())
+
+                with io.open(srt_file, 'wt', encoding='utf-8') as f:
+                    f.write(srt_data)
+
+                ext = 'srt'
+                subs[lang] = {
+                    'ext': 'srt',
+                    'data': srt_data
+                }
+
+                if new_ext == 'srt':
+                    continue
+
             self.run_ffmpeg(
                 subtitles_filename(filename, lang, ext),
                 new_file, ['-f', new_format])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index edeee1853..5e1c4525d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1800,6 +1800,59 @@ def match_filter_func(filter_str):
     return _match_func
 
 
+def parse_dfxp_time_expr(time_expr):
+    if not time_expr:
+        return 0.0
+
+    mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
+    if mobj:
+        return float(mobj.group('time_offset'))
+
+    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
+    if mobj:
+        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
+
+
+def format_srt_time(seconds):
+    (mins, secs) = divmod(seconds, 60)
+    (hours, mins) = divmod(mins, 60)
+    millisecs = (secs - int(secs)) * 1000
+    secs = int(secs)
+    return '%02d:%02d:%02d,%03d' % (hours, mins, secs, millisecs)
+
+
+def dfxp2srt(dfxp_data):
+    _x = functools.partial(xpath_with_ns, ns_map={'ttml': 'http://www.w3.org/ns/ttml'})
+
+    def parse_node(node):
+        str_or_empty = functools.partial(str_or_none, default='')
+
+        out = str_or_empty(node.text)
+
+        for child in node:
+            if child.tag == _x('ttml:br'):
+                out += '\n' + str_or_empty(child.tail)
+            elif child.tag == _x('ttml:span'):
+                out += str_or_empty(parse_node(child))
+            else:
+                out += str_or_empty(xml.etree.ElementTree.tostring(child))
+
+        return out
+
+    dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
+    out = []
+    paras = dfxp.findall(_x('.//ttml:p'))
+
+    for para, index in zip(paras, itertools.count(1)):
+        out.append('%d\n%s --> %s\n%s\n\n' % (
+            index,
+            format_srt_time(parse_dfxp_time_expr(para.attrib.get('begin'))),
+            format_srt_time(parse_dfxp_time_expr(para.attrib.get('end'))),
+            parse_node(para)))
+
+    return ''.join(out)
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From f54bab4d6777535390d64502bd20bb5a81b5468a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 22:39:50 +0600
Subject: [PATCH 0387/1515] [instagram] Improve _VALID_URL

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b020e2621..d219ceb55 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'http://instagram\.com/p/(?P<id>.*?)/'
+    _VALID_URL = r'https?://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
     _TEST = {
         'url': 'http://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',

From d2d8248f68c029472bb81eacf4aa15987729977f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 22:42:15 +0600
Subject: [PATCH 0388/1515] [instagram] Modernize

---
 youtube_dl/extractor/instagram.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index d219ceb55..65f6ca103 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -3,9 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
+from ..utils import int_or_none
 
 
 class InstagramIE(InfoExtractor):
@@ -23,8 +21,8 @@ class InstagramIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
         uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
                                          webpage, 'uploader id', fatal=False)

From b860f5dfd45f58b2c0a92cecd28c7eb05a7a2ff4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 00:22:13 +0600
Subject: [PATCH 0389/1515] [mplayer] Clarify error message

---
 youtube_dl/downloader/mplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 551c4ae94..b1ea8daa6 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -28,7 +28,7 @@ class MplayerFD(FileDownloader):
             args = [
                 'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
         else:
-            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run')
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
             return False
 
         # Download using mplayer.

From b874495b1fa339126c306df54aa316043ac0047b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 00:23:16 +0600
Subject: [PATCH 0390/1515] [mplayer] Simplify

---
 youtube_dl/downloader/mplayer.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index b1ea8daa6..0cad5a148 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -16,14 +16,10 @@ class MplayerFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = []
-        # Check for mplayer first
         if check_executable('mplayer', ['-h']):
             args = [
                 'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
                 '-dumpstream', '-dumpfile', tmpfilename, url]
-                        
-        # Check for mpv
         elif check_executable('mpv', ['-h']):
             args = [
                 'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
@@ -31,7 +27,6 @@ class MplayerFD(FileDownloader):
             self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
             return False
 
-        # Download using mplayer.
         retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
@@ -46,5 +41,5 @@ class MplayerFD(FileDownloader):
             return True
         else:
             self.to_stderr('\n')
-            self.report_error('mplayer exited with code %d' % retval)
+            self.report_error('%s exited with code %d' % (args[0], retval))
             return False

From a5ebf77d877ededed377db920faf4f31644573cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 00:25:51 +0600
Subject: [PATCH 0391/1515] [mplayer] Rename to RTSP

---
 youtube_dl/downloader/__init__.py             | 6 +++---
 youtube_dl/downloader/{mplayer.py => rtsp.py} | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename youtube_dl/downloader/{mplayer.py => rtsp.py} (97%)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 9fb66e2f7..f110830c4 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -6,7 +6,7 @@ from .f4m import F4mFD
 from .hls import HlsFD
 from .hls import NativeHlsFD
 from .http import HttpFD
-from .mplayer import MplayerFD
+from .rtsp import RtspFD
 from .rtmp import RtmpFD
 
 from ..utils import (
@@ -17,8 +17,8 @@ PROTOCOL_MAP = {
     'rtmp': RtmpFD,
     'm3u8_native': NativeHlsFD,
     'm3u8': HlsFD,
-    'mms': MplayerFD,
-    'rtsp': MplayerFD,
+    'mms': RtspFD,
+    'rtsp': RtspFD,
     'f4m': F4mFD,
 }
 
diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/rtsp.py
similarity index 97%
rename from youtube_dl/downloader/mplayer.py
rename to youtube_dl/downloader/rtsp.py
index 0cad5a148..3eb29526c 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/rtsp.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class MplayerFD(FileDownloader):
+class RtspFD(FileDownloader):
     def real_download(self, filename, info_dict):
         url = info_dict['url']
         self.report_destination(filename)

From 2a09c1b8ab348c43ece3cf246645e7335a802ca9 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 25 Apr 2015 20:41:15 +0200
Subject: [PATCH 0392/1515] [postprocessor/embedthumbnail] Fix mp3 embedding
 with avconv (fixes #5526)

---
 youtube_dl/postprocessor/embedthumbnail.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 7ba98a0ea..4868a42fd 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urlretrieve,
 )
 from ..utils import (
+    determine_ext,
     check_executable,
     encodeFilename,
     PostProcessingError,
@@ -27,7 +28,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = prepend_extension(filename, 'thumb')
+        temp_thumbnail = filename + '.' + determine_ext(info['thumbnail'])
 
         if not info.get('thumbnail'):
             raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')

From 642f23bd81ae31bb9b5647ff515db66e3023c35f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Apr 2015 22:34:28 +0200
Subject: [PATCH 0393/1515] =?UTF-8?q?[southpark]=20Use=20'=C3=B1'=20in=20t?=
 =?UTF-8?q?he=20spanish=20extractor=20name?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

IE_NAME can contain unicode characters, so it shouldn't be a problem.
---
 youtube_dl/extractor/southpark.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 77758bbed..e3b73295c 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
@@ -21,7 +22,7 @@ class SouthParkIE(MTVServicesInfoExtractor):
 
 
 class SouthParkEsIE(SouthParkIE):
-    IE_NAME = 'southpark.cc.com:espanol'
+    IE_NAME = 'southpark.cc.com:español'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
     _LANG = 'es'
 

From aa49acd15a92faa5cfc1d2876821743f86440c13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:29:41 +0600
Subject: [PATCH 0394/1515] [utils] Add `get_subprocess_encoding` and
 filename/argument decode counterparts

---
 youtube_dl/utils.py | 49 +++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5e1c4525d..7de7742e3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -371,6 +371,18 @@ def unescapeHTML(s):
         r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
 
 
+def get_subprocess_encoding():
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # For subprocess calls, encode with locale encoding
+        # Refer to http://stackoverflow.com/a/9951851/35070
+        encoding = preferredencoding()
+    else:
+        encoding = sys.getfilesystemencoding()
+    if encoding is None:
+        encoding = 'utf-8'
+    return encoding
+
+
 def encodeFilename(s, for_subprocess=False):
     """
     @param s The name of the file
@@ -382,21 +394,24 @@ def encodeFilename(s, for_subprocess=False):
     if sys.version_info >= (3, 0):
         return s
 
-    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        # Pass '' directly to use Unicode APIs on Windows 2000 and up
-        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
-        # match Windows 9x series as well. Besides, NT 4 is obsolete.)
-        if not for_subprocess:
-            return s
-        else:
-            # For subprocess calls, encode with locale encoding
-            # Refer to http://stackoverflow.com/a/9951851/35070
-            encoding = preferredencoding()
-    else:
-        encoding = sys.getfilesystemencoding()
-    if encoding is None:
-        encoding = 'utf-8'
-    return s.encode(encoding, 'ignore')
+    # Pass '' directly to use Unicode APIs on Windows 2000 and up
+    # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+    # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+    if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        return s
+
+    return s.encode(get_subprocess_encoding(), 'ignore')
+
+
+def decodeFilename(b, for_subprocess=False):
+
+    if sys.version_info >= (3, 0):
+        return b
+
+    if not isinstance(b, bytes):
+        return b
+
+    return b.decode(get_subprocess_encoding(), 'ignore')
 
 
 def encodeArgument(s):
@@ -408,6 +423,10 @@ def encodeArgument(s):
     return encodeFilename(s, True)
 
 
+def decodeArgument(b):
+    return decodeFilename(b, True)
+
+
 def decodeOption(optval):
     if optval is None:
         return optval

From cd8a07a7649cbd0feea476c89882c5c85130724f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:30:45 +0600
Subject: [PATCH 0395/1515] [downloader/common] Use decodeArgument

---
 youtube_dl/downloader/common.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index a0fc5ead0..97e755d4b 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -8,6 +8,7 @@ import time
 from ..compat import compat_str
 from ..utils import (
     encodeFilename,
+    decodeArgument,
     format_bytes,
     timeconvert,
 )
@@ -353,19 +354,15 @@ class FileDownloader(object):
         # this interface
         self._progress_hooks.append(ph)
 
-    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+    def _debug_cmd(self, args, exe=None):
         if not self.params.get('verbose', False):
             return
 
-        if exe is None:
-            exe = os.path.basename(args[0])
+        str_args = [decodeArgument(a) for a in args]
+
+        if exe is None:
+            exe = os.path.basename(str_args[0])
 
-        if subprocess_encoding:
-            str_args = [
-                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                for a in args]
-        else:
-            str_args = args
         try:
             import pipes
             shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))

From 9e105a858c2aa0089764326cc3fca808c51a9ffd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:32:54 +0600
Subject: [PATCH 0396/1515] [downloader/rtmp] Fix arguments encoding and
 simplify retry logic (Closes #5528)

---
 youtube_dl/downloader/rtmp.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index ddf5724ae..6865b5e2f 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import os
 import re
 import subprocess
-import sys
 import time
 
 from .common import FileDownloader
@@ -11,6 +10,7 @@ from ..compat import compat_str
 from ..utils import (
     check_executable,
     encodeFilename,
+    encodeArgument,
     get_exe_version,
 )
 
@@ -121,7 +121,7 @@ class RtmpFD(FileDownloader):
         # possible. This is part of rtmpdump's normal usage, AFAIK.
         basic_args = [
             'rtmpdump', '--verbose', '-r', url,
-            '-o', encodeFilename(tmpfilename, True)]
+            '-o', tmpfilename]
         if player_url is not None:
             basic_args += ['--swfVfy', player_url]
         if page_url is not None:
@@ -154,16 +154,9 @@ class RtmpFD(FileDownloader):
         if not live and continue_dl:
             args += ['--skip', '1']
 
-        if sys.platform == 'win32' and sys.version_info < (3, 0):
-            # Windows subprocess module does not actually support Unicode
-            # on Python 2.x
-            # See http://stackoverflow.com/a/9951851/35070
-            subprocess_encoding = sys.getfilesystemencoding()
-            args = [a.encode(subprocess_encoding, 'ignore') for a in args]
-        else:
-            subprocess_encoding = None
+        args = [encodeArgument(a) for a in args]
 
-        self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
+        self._debug_cmd(args, exe='rtmpdump')
 
         RD_SUCCESS = 0
         RD_FAILED = 1
@@ -180,7 +173,11 @@ class RtmpFD(FileDownloader):
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('[rtmpdump] %s bytes' % prevsize)
             time.sleep(5.0)  # This seems to be needed
-            retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED])
+            args = basic_args + ['--resume']
+            if retval == RD_FAILED:
+                args += ['--skip', '1']
+            args = [encodeArgument(a) for a in args]
+            retval = run_rtmpdump(args)
             cursize = os.path.getsize(encodeFilename(tmpfilename))
             if prevsize == cursize and retval == RD_FAILED:
                 break

From 74f8654a53ed56dd813ace704151ea575963cabb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:33:43 +0600
Subject: [PATCH 0397/1515] [downloader/external] Use encodeArgument

---
 youtube_dl/downloader/external.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1673b2382..7ca2d3143 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -2,11 +2,11 @@ from __future__ import unicode_literals
 
 import os.path
 import subprocess
-import sys
 
 from .common import FileDownloader
 from ..utils import (
     encodeFilename,
+    encodeArgument,
 )
 
 
@@ -60,17 +60,9 @@ class ExternalFD(FileDownloader):
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
-        cmd = self._make_cmd(tmpfilename, info_dict)
+        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
 
-        if sys.platform == 'win32' and sys.version_info < (3, 0):
-            # Windows subprocess module does not actually support Unicode
-            # on Python 2.x
-            # See http://stackoverflow.com/a/9951851/35070
-            subprocess_encoding = sys.getfilesystemencoding()
-            cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
-        else:
-            subprocess_encoding = None
-        self._debug_cmd(cmd, subprocess_encoding)
+        self._debug_cmd(cmd)
 
         p = subprocess.Popen(
             cmd, stderr=subprocess.PIPE)

From 06d07c400088d35096df1406a3434c996795250d Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Sun, 26 Apr 2015 14:15:29 +0200
Subject: [PATCH 0398/1515] New extractor: live.philharmoniedeparis.fr

---
 youtube_dl/extractor/__init__.py            |  1 +
 youtube_dl/extractor/philharmoniedeparis.py | 77 +++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 youtube_dl/extractor/philharmoniedeparis.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ab80fd5e0..641c45f43 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -377,6 +377,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
new file mode 100644
index 000000000..7fc2f32ab
--- /dev/null
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+    unified_strdate,
+)
+
+class PhilharmonieDeParisIE(InfoExtractor):
+    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/concert/(?P<id>\d+)(?:/|\.html)'
+    _TESTS = [{
+        'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
+        'info_dict': {
+            'id': '1032066',
+            'ext': 'mp4',
+            'title': "Week-end Bach. Passion selon saint Jean. Akademie für alte Musik Berlin, Rias Kammerchor, René Jacobs",
+            'upload_date': '20150404',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        fichier_nom = self._html_search_regex(r'\sflashvars\s*:\s*\{\s*fichier_nom\s*:\s*\'(.*?)\'\s*,', webpage, 'fichier_nom')
+
+        playlist = self._download_xml('http://live.philharmoniedeparis.fr' + fichier_nom, video_id)
+
+        concert = playlist.find('.//concert')
+
+        formats = []
+        info_dict = {
+            'id': video_id,
+            'title': concert.find('./titre').text,
+            'formats': formats,
+        }
+
+        if concert.attrib.get('heure'):
+            info_dict['timestamp'] = parse_iso8601(('%s-%s-%s%s') % (
+                concert.attrib['date'][0:4],
+                concert.attrib['date'][4:6],
+                concert.attrib['date'][6:8],
+                concert.attrib['heure']
+            ))
+        else:
+            info_dict['upload_date'] = concert.attrib['date']
+
+        fichiers = concert.find('./fichiers')
+        for fichier in fichiers.findall('./fichier'):
+            # Sometimes <ficher>s have no attributes at all. Skip them.
+            if 'url' not in fichier.attrib:
+                continue
+
+            formats.append({
+                'format_id': 'lq',
+                'url': fichiers.attrib['serveurstream'],
+                'ext': determine_ext(fichier.attrib['url']),
+                'play_path': fichier.attrib['url'],
+                'width': int_or_none(concert.attrib['largeur']),
+                'height': int_or_none(concert.attrib['hauteur']),
+                'quality': 1,
+            })
+
+            formats.append({
+                'format_id': 'hq',
+                'url': fichiers.attrib['serveurstream'],
+                'ext': determine_ext(fichier.attrib['url_hd']),
+                'play_path': fichier.attrib['url_hd'],
+                'width': int_or_none(concert.attrib['largeur_hd']),
+                'height': int_or_none(concert.attrib['hauteur_hd']),
+                'quality': 2,
+            })
+
+        return info_dict

From 4eb5c65beec671e534ad4859c073adb362e537ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Apr 2015 22:45:20 +0200
Subject: [PATCH 0399/1515] release 2015.04.26

---
 docs/supportedsites.md | 10 +++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 80e86c1b6..cc1564b7b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -174,6 +174,7 @@
  - **Gazeta**
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
+ - **Gfycat**
  - **GiantBomb**
  - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
@@ -181,7 +182,7 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in and fastvideo.in
+ - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net
  - **Goshgay**
  - **Grooveshark**
  - **Groupon**
@@ -252,6 +253,7 @@
  - **Malemotion**
  - **MDR**
  - **media.ccc.de**
+ - **MegaVideoz**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
@@ -405,6 +407,7 @@
  - **Screencast**
  - **ScreencastOMatic**
  - **ScreenwaveMedia**
+ - **SenateISVP**
  - **ServingSys**
  - **Sexu**
  - **SexyKarma**: Sexy Karma and Watch Indian Porn
@@ -427,6 +430,7 @@
  - **soundgasm**
  - **soundgasm:profile**
  - **southpark.cc.com**
+ - **southpark.cc.com:español**
  - **southpark.de**
  - **Space**
  - **SpankBang**
@@ -461,7 +465,7 @@
  - **TeamFour**
  - **TechTalks**
  - **techtv.mit.edu**
- - **TED**
+ - **ted**
  - **tegenlicht.vpro.nl**
  - **TeleBruxelles**
  - **telecinco.es**
@@ -551,7 +555,7 @@
  - **vimeo:review**: Review pages on vimeo
  - **vimeo:user**
  - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
- - **Vimple**: Vimple.ru
+ - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
  - **vk.com**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3fd0e7e56..dc7d666dd 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.17'
+__version__ = '2015.04.26'

From a01cfc2951745c36cb3e8664bacff3ff8a61d4d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 03:36:32 +0600
Subject: [PATCH 0400/1515] [philharmoniedeparis] Fix extraction and tests,
 improve, simplify

---
 youtube_dl/extractor/philharmoniedeparis.py | 89 +++++++++++----------
 1 file changed, 45 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
index 7fc2f32ab..6e60e5fe9 100644
--- a/youtube_dl/extractor/philharmoniedeparis.py
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -3,75 +3,76 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
+    float_or_none,
     int_or_none,
     parse_iso8601,
-    unified_strdate,
+    xpath_text,
 )
 
+
 class PhilharmonieDeParisIE(InfoExtractor):
-    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/concert/(?P<id>\d+)(?:/|\.html)'
+    IE_DESC = 'Philharmonie de Paris'
+    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
         'info_dict': {
             'id': '1032066',
-            'ext': 'mp4',
-            'title': "Week-end Bach. Passion selon saint Jean. Akademie für alte Musik Berlin, Rias Kammerchor, René Jacobs",
+            'ext': 'flv',
+            'title': 'md5:d1f5585d87d041d07ce9434804bc8425',
+            'timestamp': 1428179400,
             'upload_date': '20150404',
+            'duration': 6592.278,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
+    }, {
+        'url': 'http://live.philharmoniedeparis.fr/Concert/1030324.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=1030324&track=&lang=fr',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        fichier_nom = self._html_search_regex(r'\sflashvars\s*:\s*\{\s*fichier_nom\s*:\s*\'(.*?)\'\s*,', webpage, 'fichier_nom')
-
-        playlist = self._download_xml('http://live.philharmoniedeparis.fr' + fichier_nom, video_id)
-
-        concert = playlist.find('.//concert')
+        concert = self._download_xml(
+            'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=%s' % video_id,
+            video_id).find('./concert')
 
         formats = []
         info_dict = {
             'id': video_id,
-            'title': concert.find('./titre').text,
+            'title': xpath_text(concert, './titre', 'title', fatal=True),
             'formats': formats,
         }
 
-        if concert.attrib.get('heure'):
-            info_dict['timestamp'] = parse_iso8601(('%s-%s-%s%s') % (
-                concert.attrib['date'][0:4],
-                concert.attrib['date'][4:6],
-                concert.attrib['date'][6:8],
-                concert.attrib['heure']
-            ))
-        else:
-            info_dict['upload_date'] = concert.attrib['date']
-
         fichiers = concert.find('./fichiers')
+        stream = fichiers.attrib['serveurstream']
         for fichier in fichiers.findall('./fichier'):
-            # Sometimes <ficher>s have no attributes at all. Skip them.
-            if 'url' not in fichier.attrib:
-                continue
+            info_dict['duration'] = float_or_none(fichier.get('timecodefin'))
+            for quality, (format_id, suffix) in enumerate([('lq', ''), ('hq', '_hd')]):
+                format_url = fichier.get('url%s' % suffix)
+                if not format_url:
+                    continue
+                formats.append({
+                    'url': stream,
+                    'play_path': format_url,
+                    'ext': 'flv',
+                    'format_id': format_id,
+                    'width': int_or_none(concert.get('largeur%s' % suffix)),
+                    'height': int_or_none(concert.get('hauteur%s' % suffix)),
+                    'quality': quality,
+                })
+        self._sort_formats(formats)
 
-            formats.append({
-                'format_id': 'lq',
-                'url': fichiers.attrib['serveurstream'],
-                'ext': determine_ext(fichier.attrib['url']),
-                'play_path': fichier.attrib['url'],
-                'width': int_or_none(concert.attrib['largeur']),
-                'height': int_or_none(concert.attrib['hauteur']),
-                'quality': 1,
-            })
-
-            formats.append({
-                'format_id': 'hq',
-                'url': fichiers.attrib['serveurstream'],
-                'ext': determine_ext(fichier.attrib['url_hd']),
-                'play_path': fichier.attrib['url_hd'],
-                'width': int_or_none(concert.attrib['largeur_hd']),
-                'height': int_or_none(concert.attrib['hauteur_hd']),
-                'quality': 2,
-            })
+        date, hour = concert.get('date'), concert.get('heure')
+        if date and hour:
+            info_dict['timestamp'] = parse_iso8601(
+                '%s-%s-%sT%s:00' % (date[0:4], date[4:6], date[6:8], hour))
+        elif date:
+            info_dict['upload_date'] = date
 
         return info_dict

From 053c94f1b358a44789c38efd916904ce2733fd51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 15:21:51 +0600
Subject: [PATCH 0401/1515] [README] Clarify youtube-dl version when format
 selection changed to bestvideo+bestaudio/best

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7040be1ec..b4e4b5db6 100644
--- a/README.md
+++ b/README.md
@@ -270,9 +270,9 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
 
-If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From c8183e661d629bd7d86d38eda9e0013e2aad682e Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 16:01:30 +0600
Subject: [PATCH 0402/1515] [README] Document special characters escaping
 (#5538)

---
 README.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/README.md b/README.md
index b4e4b5db6..5fb992241 100644
--- a/README.md
+++ b/README.md
@@ -358,6 +358,22 @@ YouTube has switched to a new video info format in July 2011 which is not suppor
 
 YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
+### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
+
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+
+For example if your URL is https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw you should end up with following command:
+
+```youtube-dl 'https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw'```
+
+or
+
+```youtube-dl https://www.youtube.com/watch?t=48\&v=e2CXaJv0cMw```
+
+For Windows you have to use the double quotes:
+
+```youtube-dl "https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw"```
+
 ### ExtractorError: Could not find JS function u'OF'
 
 In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.

From 189ba90996d810ea00b165d20d320869efe14518 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 16:05:01 +0600
Subject: [PATCH 0403/1515] [README] Use youtube-dl test video URL

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 5fb992241..f67744f0f 100644
--- a/README.md
+++ b/README.md
@@ -362,17 +362,17 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
-For example if your URL is https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw you should end up with following command:
+For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
-```youtube-dl 'https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw'```
+```youtube-dl 'https://www.youtube.com/watch?t=4&v=BaW_jenozKc'```
 
 or
 
-```youtube-dl https://www.youtube.com/watch?t=48\&v=e2CXaJv0cMw```
+```youtube-dl https://www.youtube.com/watch?t=4\&v=BaW_jenozKc```
 
 For Windows you have to use the double quotes:
 
-```youtube-dl "https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw"```
+```youtube-dl "https://www.youtube.com/watch?t=4&v=BaW_jenozKc"```
 
 ### ExtractorError: Could not find JS function u'OF'
 

From c86b61428b127ee29793f005c5e45e321696d7e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 20:00:18 +0600
Subject: [PATCH 0404/1515] [utils] Fix another old python 2.6 kwargs issue
 (Closes #5539)

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7d15eab64..f07679c76 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -37,6 +37,7 @@ from .compat import (
     compat_chr,
     compat_html_entities,
     compat_http_client,
+    compat_kwargs,
     compat_parse_qs,
     compat_socket_create_connection,
     compat_str,
@@ -114,7 +115,7 @@ def write_json_file(obj, fn):
             'encoding': 'utf-8',
         })
 
-    tf = tempfile.NamedTemporaryFile(**args)
+    tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
 
     try:
         with tf:

From e2dc351d25d1b05b26cba654a22b0d10ed47dec9 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Mon, 27 Apr 2015 17:44:13 +0200
Subject: [PATCH 0405/1515] [escapist] Fix extractor (fixes #5090)

---
 youtube_dl/extractor/escapist.py | 156 +++++++++++++------------------
 1 file changed, 65 insertions(+), 91 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index e47f3e27a..a01a05b06 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -1,128 +1,102 @@
 from __future__ import unicode_literals
 
+import json
+
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+
 from ..utils import (
-    ExtractorError,
-    js_to_json,
-    parse_duration,
+    determine_ext,
+    clean_html,
+    qualities,
 )
 
 
+def _decrypt_config(key, string):
+    a = ''
+    i = ''
+    r = ''
+
+    while len(a) < (len(string) / 2):
+        a += key
+
+    a = a[0:int(len(string) / 2)]
+
+    t = 0
+    while t < len(string):
+        i += chr(int(string[t] + string[t + 1], 16))
+        t += 2
+
+    icko = [s for s in i]
+
+    for t, c in enumerate(a):
+        r += chr(ord(c) ^ ord(icko[t]))
+
+    return r
+
+
 class EscapistIE(InfoExtractor):
     _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
-    _USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
-        'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
+        'md5': 'c6793dbda81388f4264c1ba18684a74d',
         'info_dict': {
             'id': '6618',
             'ext': 'mp4',
             'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
-            'uploader_id': 'the-escapist-presents',
-            'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 264,
         }
-    }
+    }, {
+        'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
+        'md5': 'cf8842a8a46444d241f9a9980d7874f2',
+        'info_dict': {
+            'id': '10044',
+            'ext': 'mp4',
+            'description': 'This week, Zero Punctuation reviews Evolve.',
+            'title': 'Evolve - One vs Multiplayer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 304,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage_req = compat_urllib_request.Request(url)
-        webpage_req.add_header('User-Agent', self._USER_AGENT)
-        webpage = self._download_webpage(webpage_req, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        uploader_id = self._html_search_regex(
-            r"<h1\s+class='headline'>\s*<a\s+href='/videos/view/(.*?)'",
-            webpage, 'uploader ID', fatal=False)
-        uploader = self._html_search_regex(
-            r"<h1\s+class='headline'>(.*?)</a>",
-            webpage, 'uploader', fatal=False)
-        description = self._html_search_meta('description', webpage)
-        duration = parse_duration(self._html_search_meta('duration', webpage))
+        imsVideo = json.loads(self._search_regex(r'imsVideo\.play\(([^\)]+)\);', webpage, 'imsVideo'))
+        video_id = imsVideo['videoID']
+        key = imsVideo['hash']
 
-        raw_title = self._html_search_meta('title', webpage, fatal=True)
-        title = raw_title.partition(' : ')[2]
-
-        config_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'''(?x)
-            (?:
-                <param\s+name="flashvars".*?\s+value="config=|
-                flashvars=&quot;config=
-            )
-            (https?://[^"&]+)
-            ''',
-            webpage, 'config URL'))
+        quality = qualities(['lq', 'hq', 'hd'])
 
         formats = []
-        ad_formats = []
+        for q in ['lq', 'hq', 'hd']:
+            config_req = compat_urllib_request.Request('http://www.escapistmagazine.com/videos/'
+                'vidconfig.php?videoID=%s&hash=%s&quality=%s' % (video_id, key, 'mp4_' + q))
+            config_req.add_header('Referer', url)
+            config = self._download_webpage(config_req, video_id, 'Downloading video config ' + q.upper())
 
-        def _add_format(name, cfg_url, quality):
-            cfg_req = compat_urllib_request.Request(cfg_url)
-            cfg_req.add_header('User-Agent', self._USER_AGENT)
-            config = self._download_json(
-                cfg_req, video_id,
-                'Downloading ' + name + ' configuration',
-                'Unable to download ' + name + ' configuration',
-                transform_source=js_to_json)
+            data = json.loads(_decrypt_config(key, config))
 
-            playlist = config['playlist']
-            for p in playlist:
-                if p.get('eventCategory') == 'Video':
-                    ar = formats
-                elif p.get('eventCategory') == 'Video Postroll':
-                    ar = ad_formats
-                else:
-                    continue
+            title = clean_html(data['videoData']['title'])
+            duration = data['videoData']['duration'] / 1000
 
-                ar.append({
-                    'url': p['url'],
-                    'format_id': name,
-                    'quality': quality,
-                    'http_headers': {
-                        'User-Agent': self._USER_AGENT,
-                    },
-                })
+            for i, v in enumerate(data['files']['videos']):
 
-        _add_format('normal', config_url, quality=0)
-        hq_url = (config_url +
-                  ('&hq=1' if '?' in config_url else config_url + '?hq=1'))
-        try:
-            _add_format('hq', hq_url, quality=1)
-        except ExtractorError:
-            pass  # That's fine, we'll just use normal quality
-        self._sort_formats(formats)
+                formats.append({
+                    'url': v,
+                    'format_id': determine_ext(v) + '_' + q + str(i),
+                    'quality': quality(q),
+                    })
 
-        if '/escapist/sales-marketing/' in formats[-1]['url']:
-            raise ExtractorError('This IP address has been blocked by The Escapist', expected=True)
 
-        res = {
+        return {
             'id': video_id,
             'formats': formats,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': description,
+            'description': self._og_search_description(webpage),
             'duration': duration,
         }
-
-        if self._downloader.params.get('include_ads') and ad_formats:
-            self._sort_formats(ad_formats)
-            ad_res = {
-                'id': '%s-ad' % video_id,
-                'title': '%s (Postroll)' % title,
-                'formats': ad_formats,
-            }
-            return {
-                '_type': 'playlist',
-                'entries': [res, ad_res],
-                'title': title,
-                'id': video_id,
-            }
-
-        return res

From 290a5a8d85e8f7514099fa305c8f0b26d1b35a78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 22:17:51 +0600
Subject: [PATCH 0406/1515] [escapist] Fix imsVideo regex (#5090)

---
 youtube_dl/extractor/escapist.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index a01a05b06..600ebf078 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -65,7 +65,10 @@ class EscapistIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        imsVideo = json.loads(self._search_regex(r'imsVideo\.play\(([^\)]+)\);', webpage, 'imsVideo'))
+        imsVideo = self._parse_json(
+            self._search_regex(
+                r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
+            video_id)
         video_id = imsVideo['videoID']
         key = imsVideo['hash']
 

From e206740fd7454bb03e2819365a257bce59f76162 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 23:44:05 +0600
Subject: [PATCH 0407/1515] [moniker] Capture and output error message (#5541)

---
 youtube_dl/extractor/moniker.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 5de719bdc..801abe0e1 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import ExtractorError
 
 
 class MonikerIE(InfoExtractor):
@@ -40,6 +41,12 @@ class MonikerIE(InfoExtractor):
         video_id = self._match_id(url)
         orig_webpage = self._download_webpage(url, video_id)
 
+        error = self._search_regex(
+            r'class="err">([^<]+)<', orig_webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
+
         fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
         data = dict(fields)
 

From 2419a376b911df61bec6dae7178fc342a4e218f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 23:46:16 +0600
Subject: [PATCH 0408/1515] [moniker] Check not found error (#5541)

---
 youtube_dl/extractor/moniker.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 801abe0e1..88dcd4f73 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -41,6 +41,9 @@ class MonikerIE(InfoExtractor):
         video_id = self._match_id(url)
         orig_webpage = self._download_webpage(url, video_id)
 
+        if '>File Not Found<' in orig_webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         error = self._search_regex(
             r'class="err">([^<]+)<', orig_webpage, 'error', default=None)
         if error:

From 3a0f0c263a9c6e2a3e81fc97f59a5960fd5ee994 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Apr 2015 09:11:18 +0200
Subject: [PATCH 0409/1515] release 2015.04.28

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cc1564b7b..d1d676cdd 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -340,6 +340,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
+ - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
  - **Pladform**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dc7d666dd..b88ea85e8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.26'
+__version__ = '2015.04.28'

From 10831b5ec9158f2de5b16b66520d9dde02b97d77 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Tue, 28 Apr 2015 14:56:48 +0200
Subject: [PATCH 0410/1515] [vimeo] Fix redirection

---
 youtube_dl/extractor/vimeo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9027f9dd6..398ca67c1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -223,6 +223,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
         orig_url = url
         if mobj.group('pro') or mobj.group('player'):
             url = 'https://player.vimeo.com/video/' + video_id
+        else:
+            url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)

From 2edce52584ecd14ee626c6a1b6bbaeabb3985cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 28 Apr 2015 15:05:41 +0200
Subject: [PATCH 0411/1515] [vimeo] Fix password protected videos again (#5082)

Since they have changed again to the previous format, I've modified the regex to match both formats.
---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 398ca67c1..f300c7ca4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -177,7 +177,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -441,7 +441,7 @@ class VimeoChannelIE(InfoExtractor):
             name="([^"]+)"\s+
             value="([^"]*)"
             ''', login_form))
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)

From 39b62db1160f5a4770348f1d01daeb0ce049c28c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 28 Apr 2015 23:07:56 +0800
Subject: [PATCH 0412/1515] [youtube] Catch more alert messages (closes #5074)

---
 youtube_dl/extractor/youtube.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 07c0f6ef9..0869c9fd4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1291,12 +1291,22 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         page = self._download_webpage(url, playlist_id)
         more_widget_html = content_html = page
 
-        # Check if the playlist exists or is private
-        if re.search(r'<div class="yt-alert-message">[^<]*?(The|This) playlist (does not exist|is private)[^<]*?</div>', page) is not None:
-            raise ExtractorError(
-                'The playlist doesn\'t exist or is private, use --username or '
-                '--netrc to access it.',
-                expected=True)
+        for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
+            match = match.strip()
+            # Check if the playlist exists or is private
+            if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):
+                raise ExtractorError(
+                    'The playlist doesn\'t exist or is private, use --username or '
+                    '--netrc to access it.',
+                    expected=True)
+            elif re.match(r'[^<]*Invalid parameters[^<]*', match):
+                raise ExtractorError(
+                    'Invalid parameters. Maybe URL is incorrect.',
+                    expected=True)
+            elif re.match(r'[^<]*Choose your language[^<]*', match):
+                continue
+            else:
+                self.report_warning('Youtube gives an alert message: ' + match)
 
         # Extract the video ids from the playlist pages
         ids = []

From e70c7568c0226449190b57b988c1df10bf6508db Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 00:01:09 +0800
Subject: [PATCH 0413/1515] [testtube] Detect Youtube iframes (fixes #4867)

---
 youtube_dl/extractor/testtube.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index 6a7b5e49d..cbada136c 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -22,12 +22,30 @@ class TestTubeIE(InfoExtractor):
             'uploader': 'DNews',
             'uploader_id': 'dnews',
         },
+    }, {
+        'url': 'https://testtube.com/iflscience/insane-jet-ski-flipping',
+        'info_dict': {
+            'id': 'fAGfJ4YjVus',
+            'ext': 'mp4',
+            'title': 'Flipping Jet-Ski Skills | Outrageous Acts of Science',
+            'uploader': 'Science Channel',
+            'uploader_id': 'ScienceChannel',
+            'upload_date': '20150203',
+            'description': 'md5:e61374030015bae1d2e22f096d4769d6',
+        }
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
+
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+            webpage, 'youtube iframe', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube', video_id=display_id)
+
         video_id = self._search_regex(
             r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
             webpage, 'video ID')

From ec7c1e85e0b26e67752bbd3d25b8d5ed23ae0992 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 00:24:58 +0800
Subject: [PATCH 0414/1515] [testtube] Fix test case 1

Seems the site now provides webm with higher bitrates
---
 youtube_dl/extractor/testtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index cbada136c..26655d690 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -15,7 +15,7 @@ class TestTubeIE(InfoExtractor):
             'id': '60163',
             'display_id': '5-weird-ways-plants-can-eat-animals',
             'duration': 275,
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': '5 Weird Ways Plants Can Eat Animals',
             'description': 'Why have some plants evolved to eat meat?',
             'thumbnail': 're:^https?://.*\.jpg$',

From 9d8ba307ef18e737efc0be5164b53e0546288a4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 04:03:07 +0800
Subject: [PATCH 0415/1515] [yourupload] Fix extraction

---
 youtube_dl/extractor/yourupload.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dl/extractor/yourupload.py
index 40fc4165f..2064f4a69 100644
--- a/youtube_dl/extractor/yourupload.py
+++ b/youtube_dl/extractor/yourupload.py
@@ -16,7 +16,7 @@ class YourUploadIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://yourupload.com/watch/14i14h',
-            'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
+            'md5': '5e2c63385454c557f97c4c4131a393cd',
             'info_dict': {
                 'id': '14i14h',
                 'ext': 'mp4',
@@ -38,16 +38,16 @@ class YourUploadIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        url = 'http://embed.yucache.net/{0:}'.format(video_id)
-        webpage = self._download_webpage(url, video_id)
+        embed_url = 'http://embed.yucache.net/{0:}'.format(video_id)
+        webpage = self._download_webpage(embed_url, video_id)
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        url = self._og_search_video_url(webpage)
+        video_url = self._og_search_video_url(webpage)
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
         }]
 
         return {
@@ -55,4 +55,7 @@ class YourUploadIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
+            'http_headers': {
+                'Referer': embed_url,
+            },
         }

From cbbece96a25443dbf5cb66819883efa2e9b66de4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 04:05:14 +0800
Subject: [PATCH 0416/1515] [yourupload] Simplify

---
 youtube_dl/extractor/yourupload.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dl/extractor/yourupload.py
index 2064f4a69..4e25d6f22 100644
--- a/youtube_dl/extractor/yourupload.py
+++ b/youtube_dl/extractor/yourupload.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -35,25 +33,19 @@ class YourUploadIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         embed_url = 'http://embed.yucache.net/{0:}'.format(video_id)
         webpage = self._download_webpage(embed_url, video_id)
 
         title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
         video_url = self._og_search_video_url(webpage)
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
+            'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
                 'Referer': embed_url,

From 5456d78f0c177bbdd3ad0b6221ff95b706104441 Mon Sep 17 00:00:00 2001
From: zouhair <zouhair@users.noreply.github.com>
Date: Wed, 29 Apr 2015 10:07:49 -0400
Subject: [PATCH 0417/1515] Typo "incompatible" instead of "uncompatible"

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 977141881..827c88e0d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1384,7 +1384,7 @@ class YoutubeDL(object):
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         filename = os.path.splitext(filename)[0] + '.mkv'
-                        self.report_warning('You have requested formats uncompatible for merge. '
+                        self.report_warning('You have requested formats incompatible for merge. '
                                             'The formats will be merged into mkv')
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(

From cf0649f8b76879f9d32ceab91e87d659ea8ec398 Mon Sep 17 00:00:00 2001
From: zouhair <zouhair@users.noreply.github.com>
Date: Wed, 29 Apr 2015 11:03:10 -0400
Subject: [PATCH 0418/1515] Typo: twice "the the" to "the"

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 71230323c..48f58aae3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -47,7 +47,7 @@ class InfoExtractor(object):
     information possibly downloading the video to the file system, among
     other possible outcomes.
 
-    The type field determines the the type of the result.
+    The type field determines the type of the result.
     By far the most common value (and the default if _type is missing) is
     "video", which indicates a single video.
 
@@ -572,7 +572,7 @@ class InfoExtractor(object):
 
     def _get_login_info(self):
         """
-        Get the the login info as (username, password)
+        Get the login info as (username, password)
         It will look in the netrc file using the _NETRC_MACHINE value
         If there's no info available, return (None, None)
         """

From 9ee53a49f0f040f07e0f675e2077f22ace2acefe Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 04:18:53 +0800
Subject: [PATCH 0419/1515] [YouPorn] Fix extractor

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 6abe72f73..4ba7c36db 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -47,7 +47,7 @@ class YouPornIE(InfoExtractor):
 
         # Get JSON parameters
         json_params = self._search_regex(
-            [r'var\s+videoJa?son\s*=\s*({.+?});',
+            [r'videoJa?son\s*=\s*({.+})',
              r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
             webpage, 'JSON parameters')
         try:

From bb865f3a5e7290f0da43d180f068f8715c39e3ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 00:24:24 +0800
Subject: [PATCH 0420/1515] [niconico] Fix extraction and update tests (closes
 #5511)

---
 youtube_dl/extractor/niconico.py | 72 +++++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index ddec7b338..0ca046ac2 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -14,7 +14,9 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
-    unified_strdate,
+    parse_iso8601,
+    xpath_text,
+    determine_ext,
 )
 
 
@@ -32,6 +34,7 @@ class NiconicoIE(InfoExtractor):
             'uploader': 'takuya0301',
             'uploader_id': '2698420',
             'upload_date': '20131123',
+            'timestamp': 1385182762,
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },
@@ -46,16 +49,31 @@ class NiconicoIE(InfoExtractor):
             'id': 'nm14296458',
             'ext': 'swf',
             'title': '【鏡音リン】Dance on media【オリジナル】take2!',
-            'description': 'md5:',
+            'description': 'md5:689f066d74610b3b22e0f1739add0f58',
             'uploader': 'りょうた',
             'uploader_id': '18822557',
             'upload_date': '20110429',
+            'timestamp': 1304065916,
             'duration': 209,
         },
         'params': {
             'username': 'ydl.niconico@gmail.com',
             'password': 'youtube-dl',
         },
+    }, {
+        # 'video exists but is marked as "deleted"
+        'url': 'http://www.nicovideo.jp/watch/sm10000',
+        'md5': '38e53c9aad548f3ecf01ca7680b59b08',
+        'info_dict': {
+            'id': 'sm10000',
+            'ext': 'unknown_video',
+            'description': 'deleted',
+            'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
+        },
+        'params': {
+            'username': 'ydl.niconico@gmail.com',
+            'password': 'youtube-dl',
+        }
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -95,9 +113,10 @@ class NiconicoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Get video webpage. We are not actually interested in it, but need
-        # the cookies in order to be able to download the info webpage
-        self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
+        # Get video webpage. We are not actually interested in it for normal
+        # cases, but need the cookies in order to be able to download the
+        # info webpage
+        webpage = self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
 
         video_info = self._download_xml(
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
@@ -127,22 +146,34 @@ class NiconicoIE(InfoExtractor):
                 flv_info_request, video_id,
                 note='Downloading flv info', errnote='Unable to download flv info')
 
-        if 'deleted=' in flv_info_webpage:
-            raise ExtractorError('The video has been deleted.',
-                                 expected=True)
-        video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
+        flv_info = compat_urlparse.parse_qs(flv_info_webpage)
+        if 'url' not in flv_info:
+            if 'deleted' in flv_info:
+                raise ExtractorError('The video has been deleted.',
+                                     expected=True)
+            else:
+                raise ExtractorError('Unable to find video URL')
+
+        video_real_url = flv_info['url'][0]
 
         # Start extracting information
-        title = video_info.find('.//title').text
-        extension = video_info.find('.//movie_type').text
+        title = xpath_text(video_info, './/title')
+        if not title:
+            title = self._html_search_regex(
+                r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
+                webpage, 'video title')
+
+        extension = xpath_text(video_info, './/movie_type')
+        if not extension:
+            extension = determine_ext(video_real_url)
         video_format = extension.upper()
-        thumbnail = video_info.find('.//thumbnail_url').text
-        description = video_info.find('.//description').text
-        upload_date = unified_strdate(video_info.find('.//first_retrieve').text.split('+')[0])
-        view_count = int_or_none(video_info.find('.//view_counter').text)
-        comment_count = int_or_none(video_info.find('.//comment_num').text)
-        duration = parse_duration(video_info.find('.//length').text)
-        webpage_url = video_info.find('.//watch_url').text
+        thumbnail = xpath_text(video_info, './/thumbnail_url')
+        description = xpath_text(video_info, './/description')
+        timestamp = parse_iso8601(xpath_text(video_info, './/first_retrieve'))
+        view_count = int_or_none(xpath_text(video_info, './/view_counter'))
+        comment_count = int_or_none(xpath_text(video_info, './/comment_num'))
+        duration = parse_duration(xpath_text(video_info, './/length'))
+        webpage_url = xpath_text(video_info, './/watch_url')
 
         if video_info.find('.//ch_id') is not None:
             uploader_id = video_info.find('.//ch_id').text
@@ -153,7 +184,7 @@ class NiconicoIE(InfoExtractor):
         else:
             uploader_id = uploader = None
 
-        return {
+        ret = {
             'id': video_id,
             'url': video_real_url,
             'title': title,
@@ -162,13 +193,14 @@ class NiconicoIE(InfoExtractor):
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'uploader_id': uploader_id,
             'view_count': view_count,
             'comment_count': comment_count,
             'duration': duration,
             'webpage_url': webpage_url,
         }
+        return dict((k, v) for k, v in ret.items() if v is not None)
 
 
 class NiconicoPlaylistIE(InfoExtractor):

From 59d814f79341341e6390392a09e628ee12a6f18d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 00:47:52 +0800
Subject: [PATCH 0421/1515] [niconico] Remove credentials from tests and
 enhance title extraction

All test videos can be downloaded without username and password now.
---
 youtube_dl/extractor/niconico.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 0ca046ac2..a9b770cb3 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -38,13 +38,10 @@ class NiconicoIE(InfoExtractor):
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },
-        'params': {
-            'username': 'ydl.niconico@gmail.com',
-            'password': 'youtube-dl',
-        },
     }, {
+        # File downloaded with and without credentials are different, so omit
+        # the md5 field
         'url': 'http://www.nicovideo.jp/watch/nm14296458',
-        'md5': '8db08e0158457cf852a31519fceea5bc',
         'info_dict': {
             'id': 'nm14296458',
             'ext': 'swf',
@@ -56,10 +53,6 @@ class NiconicoIE(InfoExtractor):
             'timestamp': 1304065916,
             'duration': 209,
         },
-        'params': {
-            'username': 'ydl.niconico@gmail.com',
-            'password': 'youtube-dl',
-        },
     }, {
         # 'video exists but is marked as "deleted"
         'url': 'http://www.nicovideo.jp/watch/sm10000',
@@ -70,10 +63,6 @@ class NiconicoIE(InfoExtractor):
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
         },
-        'params': {
-            'username': 'ydl.niconico@gmail.com',
-            'password': 'youtube-dl',
-        }
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -158,6 +147,8 @@ class NiconicoIE(InfoExtractor):
 
         # Start extracting information
         title = xpath_text(video_info, './/title')
+        if not title:
+            title = self._og_search_title(webpage, default=None)
         if not title:
             title = self._html_search_regex(
                 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',

From b2e8e7dab567ed9b27817c5dd0cf173bc7fb8cfa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 02:24:05 +0800
Subject: [PATCH 0422/1515] [niconico] Try to extract all optional fields from
 various sources

---
 youtube_dl/extractor/niconico.py | 57 ++++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a9b770cb3..dd16d0042 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import datetime
 
 from .common import InfoExtractor
 from ..compat import (
@@ -55,13 +56,16 @@ class NiconicoIE(InfoExtractor):
         },
     }, {
         # 'video exists but is marked as "deleted"
+        # md5 is unstable
         'url': 'http://www.nicovideo.jp/watch/sm10000',
-        'md5': '38e53c9aad548f3ecf01ca7680b59b08',
         'info_dict': {
             'id': 'sm10000',
             'ext': 'unknown_video',
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
+            'upload_date': '20071224',
+            'timestamp': 1198527840,  # timestamp field has different value if logged in
+            'duration': 304,
         },
     }]
 
@@ -154,17 +158,59 @@ class NiconicoIE(InfoExtractor):
                 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
                 webpage, 'video title')
 
+        watch_api_data_string = self._html_search_regex(
+            r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>',
+            webpage, 'watch api data', default=None)
+        watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {}
+        video_detail = watch_api_data.get('videoDetail', {})
+
         extension = xpath_text(video_info, './/movie_type')
         if not extension:
             extension = determine_ext(video_real_url)
         video_format = extension.upper()
-        thumbnail = xpath_text(video_info, './/thumbnail_url')
+
+        thumbnail = (
+            xpath_text(video_info, './/thumbnail_url') or
+            self._html_search_meta('image', webpage, 'thumbnail', default=None) or
+            video_detail.get('thumbnail'))
+
         description = xpath_text(video_info, './/description')
+
         timestamp = parse_iso8601(xpath_text(video_info, './/first_retrieve'))
+        if not timestamp:
+            match = self._html_search_meta('datePublished', webpage, 'date published', default=None)
+            if match:
+                timestamp = parse_iso8601(match.replace('+', ':00+'))
+        if not timestamp and video_detail.get('postedAt'):
+            timestamp = parse_iso8601(
+                video_detail['postedAt'].replace('/', '-'),
+                delimiter=' ', timezone=datetime.timedelta(hours=9))
+
         view_count = int_or_none(xpath_text(video_info, './/view_counter'))
+        if not view_count:
+            match = self._html_search_regex(
+                r'>Views: <strong[^>]*>([^<]+)</strong>',
+                webpage, 'view count', default=None)
+            if match:
+                view_count = int_or_none(match.replace(',', ''))
+        view_count = view_count or video_detail.get('viewCount')
+
         comment_count = int_or_none(xpath_text(video_info, './/comment_num'))
-        duration = parse_duration(xpath_text(video_info, './/length'))
-        webpage_url = xpath_text(video_info, './/watch_url')
+        if not comment_count:
+            match = self._html_search_regex(
+                r'>Comments: <strong[^>]*>([^<]+)</strong>',
+                webpage, 'comment count', default=None)
+            if match:
+                comment_count = int_or_none(match.replace(',', ''))
+        comment_count = comment_count or video_detail.get('commentCount')
+
+        duration = (parse_duration(
+            xpath_text(video_info, './/length') or
+            self._html_search_meta(
+                'video:duration', webpage, 'video duration', default=None)) or
+            video_detail.get('length'))
+
+        webpage_url = xpath_text(video_info, './/watch_url') or url
 
         if video_info.find('.//ch_id') is not None:
             uploader_id = video_info.find('.//ch_id').text
@@ -175,7 +221,7 @@ class NiconicoIE(InfoExtractor):
         else:
             uploader_id = uploader = None
 
-        ret = {
+        return {
             'id': video_id,
             'url': video_real_url,
             'title': title,
@@ -191,7 +237,6 @@ class NiconicoIE(InfoExtractor):
             'duration': duration,
             'webpage_url': webpage_url,
         }
-        return dict((k, v) for k, v in ret.items() if v is not None)
 
 
 class NiconicoPlaylistIE(InfoExtractor):

From 965cb8d530e4ced61a9bc42530f7f91b67c709e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Apr 2015 22:46:19 +0200
Subject: [PATCH 0423/1515] [escapist] pep8 fixes

---
 youtube_dl/extractor/escapist.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 600ebf078..8facf1185 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -76,7 +76,8 @@ class EscapistIE(InfoExtractor):
 
         formats = []
         for q in ['lq', 'hq', 'hd']:
-            config_req = compat_urllib_request.Request('http://www.escapistmagazine.com/videos/'
+            config_req = compat_urllib_request.Request(
+                'http://www.escapistmagazine.com/videos/'
                 'vidconfig.php?videoID=%s&hash=%s&quality=%s' % (video_id, key, 'mp4_' + q))
             config_req.add_header('Referer', url)
             config = self._download_webpage(config_req, video_id, 'Downloading video config ' + q.upper())
@@ -92,8 +93,7 @@ class EscapistIE(InfoExtractor):
                     'url': v,
                     'format_id': determine_ext(v) + '_' + q + str(i),
                     'quality': quality(q),
-                    })
-
+                })
 
         return {
             'id': video_id,

From 8dd5418803a25de89d08cdb9d32f80f71c5d6c47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Apr 2015 22:53:18 +0200
Subject: [PATCH 0424/1515] Make 'best' format only match non-DASH formats
 (closes #5554)

Otherwise it's impossible to only download non-DASH formats, for example `best[height=?480]/best` would download a DASH video if it's the only one with height=480, instead for falling back to the second format specifier.
For audio only urls (soundcloud, bandcamp ...), the best audio will be downloaded as before.
---
 test/test_YoutubeDL.py  | 4 ++--
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bb4a65ee1..82b827536 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -237,7 +237,7 @@ class TestFormatSelection(unittest.TestCase):
             f2['url'] = 'url:' + f2id
 
             info_dict = _make_result([f1, f2], extractor='youtube')
-            ydl = YDL()
+            ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
@@ -245,7 +245,7 @@ class TestFormatSelection(unittest.TestCase):
             self.assertEqual(downloaded['format_id'], f1id)
 
             info_dict = _make_result([f2, f1], extractor='youtube')
-            ydl = YDL()
+            ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 827c88e0d..eee9c0154 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -915,7 +915,14 @@ class YoutubeDL(object):
             return None
 
         if format_spec == 'best' or format_spec is None:
-            return available_formats[-1]
+            audiovideo_formats = [
+                f for f in available_formats
+                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+            if audiovideo_formats:
+                return audiovideo_formats[-1]
+            # for audio only urls, 'best' selects the best audio format
+            elif all(f.get('acodec') != 'none' for f in available_formats):
+                return available_formats[-1]
         elif format_spec == 'worst':
             audiovideo_formats = [
                 f for f in available_formats

From 621ffe7bf420aa1a227e823edf2f1acbc67660d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 17:05:02 +0800
Subject: [PATCH 0425/1515] [niconico] Fix so* video extraction (fixes #4874)
 (#2087)

---
 youtube_dl/extractor/niconico.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index dd16d0042..3cecebf95 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -67,6 +67,18 @@ class NiconicoIE(InfoExtractor):
             'timestamp': 1198527840,  # timestamp field has different value if logged in
             'duration': 304,
         },
+    }, {
+        'url': 'http://www.nicovideo.jp/watch/so22543406',
+        'info_dict': {
+            'id': '1388129933',
+            'ext': 'mp4',
+            'title': '【第1回】RADIOアニメロミックス ラブライブ！～のぞえりRadio Garden～',
+            'description': 'md5:b27d224bb0ff53d3c8269e9f8b561cf1',
+            'timestamp': 1388851200,
+            'upload_date': '20140104',
+            'uploader': 'アニメロチャンネル',
+            'uploader_id': '312',
+        }
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -109,7 +121,10 @@ class NiconicoIE(InfoExtractor):
         # Get video webpage. We are not actually interested in it for normal
         # cases, but need the cookies in order to be able to download the
         # info webpage
-        webpage = self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
+        webpage, handle = self._download_webpage_handle(
+            'http://www.nicovideo.jp/watch/' + video_id, video_id)
+        if video_id.startswith('so'):
+            video_id = self._match_id(handle.geturl())
 
         video_info = self._download_xml(
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,

From c4a21bc9db1868e8be114f496899f6786b9982ec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 18:23:35 +0800
Subject: [PATCH 0426/1515] [bilibili] Extract multipart videos (closes #3250)

---
 youtube_dl/extractor/bilibili.py | 74 +++++++++++++++++++++-----------
 1 file changed, 49 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 904d9a8b4..7ca835e31 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,18 +15,25 @@ from ..utils import (
 class BiliBiliIE(InfoExtractor):
     _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>[0-9]+)/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '2c301e4dab317596e837c3e7633e7d86',
         'info_dict': {
-            'id': '1074402',
+            'id': '1074402_part1',
             'ext': 'flv',
             'title': '【金坷垃】金泡沫',
             'duration': 308,
             'upload_date': '20140420',
             'thumbnail': 're:^https?://.+\.jpg',
         },
-    }
+    }, {
+        'url': 'http://www.bilibili.com/video/av1041170/',
+        'info_dict': {
+            'id': '1041170',
+            'title': '【BD1080P】刀语【诸神&异域】',
+        },
+        'playlist_count': 9,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -57,19 +65,14 @@ class BiliBiliIE(InfoExtractor):
 
         cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
 
+        entries = []
+
         lq_doc = self._download_xml(
             'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
             video_id,
             note='Downloading LQ video info'
         )
-        lq_durl = lq_doc.find('./durl')
-        formats = [{
-            'format_id': 'lq',
-            'quality': 1,
-            'url': lq_durl.find('./url').text,
-            'filesize': int_or_none(
-                lq_durl.find('./size'), get_attr='text'),
-        }]
+        lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(
             'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
@@ -77,23 +80,44 @@ class BiliBiliIE(InfoExtractor):
             note='Downloading HQ video info',
             fatal=False,
         )
-        if hq_doc is not False:
-            hq_durl = hq_doc.find('./durl')
-            formats.append({
-                'format_id': 'hq',
-                'quality': 2,
-                'ext': 'flv',
-                'url': hq_durl.find('./url').text,
+        hq_durls = hq_doc.findall('./durl') if hq_doc is not False else itertools.repeat(None)
+
+        assert len(lq_durls) == len(hq_durls)
+
+        i = 1
+        for lq_durl, hq_durl in zip(lq_durls, hq_durls):
+            formats = [{
+                'format_id': 'lq',
+                'quality': 1,
+                'url': lq_durl.find('./url').text,
                 'filesize': int_or_none(
-                    hq_durl.find('./size'), get_attr='text'),
+                    lq_durl.find('./size'), get_attr='text'),
+            }]
+            if hq_durl:
+                formats.append({
+                    'format_id': 'hq',
+                    'quality': 2,
+                    'ext': 'flv',
+                    'url': hq_durl.find('./url').text,
+                    'filesize': int_or_none(
+                        hq_durl.find('./size'), get_attr='text'),
+                })
+            self._sort_formats(formats)
+
+            entries.append({
+                'id': '%s_part%d' % (video_id, i),
+                'title': title,
+                'formats': formats,
+                'duration': duration,
+                'upload_date': upload_date,
+                'thumbnail': thumbnail,
             })
 
-        self._sort_formats(formats)
+            i += 1
+
         return {
+            '_type': 'multi_video',
+            'entries': entries,
             'id': video_id,
-            'title': title,
-            'formats': formats,
-            'duration': duration,
-            'upload_date': upload_date,
-            'thumbnail': thumbnail,
+            'title': title
         }

From f7f1df1d82d556c0726898b9de2f7f3824c1be5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 22:37:41 +0800
Subject: [PATCH 0427/1515] [VeeHD] Enhance extraction and fix tests (fixes
 #4965)

---
 youtube_dl/extractor/veehd.py | 36 +++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 96353f525..7fdeb784d 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -17,7 +17,9 @@ from ..utils import (
 class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
-    _TEST = {
+    # Seems VeeHD videos have multiple copies on several servers, all of
+    # whom have different MD5 checksums, so omit md5 field in all tests
+    _TESTS = [{
         'url': 'http://veehd.com/video/4639434_Solar-Sinter',
         'info_dict': {
             'id': '4639434',
@@ -26,7 +28,26 @@ class VeeHDIE(InfoExtractor):
             'uploader_id': 'VideoEyes',
             'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
         },
-    }
+        'skip': 'Video deleted',
+    }, {
+        'url': 'http://veehd.com/video/4905758_Elysian-Fields-Channeling',
+        'info_dict': {
+            'id': '4905758',
+            'ext': 'mp4',
+            'title': 'Elysian Fields - Channeling',
+            'description': 'md5:360e4e95fdab58aefbea0f2a19e5604b',
+            'uploader_id': 'spotted',
+        }
+    }, {
+        'url': 'http://veehd.com/video/4665804_Tell-No-One-Ne-le-dis-a-personne-2006-French-EngSoftSubs-Re-Up',
+        'info_dict': {
+            'id': '4665804',
+            'ext': 'avi',
+            'title': 'Tell No One (Ne le dis a personne) 2006 French(EngSoftSubs) Re-Up',
+            'description': 'md5:d660cca685549776f37165e9a10b60ba',
+            'uploader_id': 'belial2549',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -48,13 +69,21 @@ class VeeHDIE(InfoExtractor):
         player_page = self._download_webpage(
             player_url, video_id, 'Downloading player page')
 
+        video_url = None
+
         config_json = self._search_regex(
             r'value=\'config=({.+?})\'', player_page, 'config json', default=None)
 
         if config_json:
             config = json.loads(config_json)
             video_url = compat_urlparse.unquote(config['clip']['url'])
-        else:
+
+        if not video_url:
+            video_url = self._html_search_regex(
+                r'<embed[^>]+type="video/divx"[^>]+src="([^"]+)"',
+                player_page, 'video url', default=None)
+
+        if not video_url:
             iframe_src = self._search_regex(
                 r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
             iframe_url = 'http://veehd.com/%s' % iframe_src
@@ -82,7 +111,6 @@ class VeeHDIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'url': video_url,
-            'ext': 'mp4',
             'uploader_id': uploader_id,
             'thumbnail': thumbnail,
             'description': description,

From e01c56f9e12d7abb2e3b548818a689146092806e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 21:06:51 +0600
Subject: [PATCH 0428/1515] [YoutubeDL] Generalize best/worst format match
 behavior

---
 youtube_dl/YoutubeDL.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eee9c0154..9d4a2dce8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -914,22 +914,16 @@ class YoutubeDL(object):
         if not available_formats:
             return None
 
-        if format_spec == 'best' or format_spec is None:
+        if format_spec in ['best', 'worst', None]:
+            format_idx = 0 if format_spec == 'worst' else -1
             audiovideo_formats = [
                 f for f in available_formats
                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
             if audiovideo_formats:
-                return audiovideo_formats[-1]
-            # for audio only urls, 'best' selects the best audio format
+                return audiovideo_formats[format_idx]
+            # for audio only urls, select the best/worst audio format
             elif all(f.get('acodec') != 'none' for f in available_formats):
-                return available_formats[-1]
-        elif format_spec == 'worst':
-            audiovideo_formats = [
-                f for f in available_formats
-                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
-            if audiovideo_formats:
-                return audiovideo_formats[0]
-            return available_formats[0]
+                return available_formats[format_idx]
         elif format_spec == 'bestaudio':
             audio_formats = [
                 f for f in available_formats

From cd298882cd6f8ed2571dd372f684ec17e992fd9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 21:25:17 +0600
Subject: [PATCH 0429/1515] [vporn] Fix metadata extraction (#5560)

---
 youtube_dl/extractor/vporn.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/vporn.py b/youtube_dl/extractor/vporn.py
index 2d23effcc..04ed0b381 100644
--- a/youtube_dl/extractor/vporn.py
+++ b/youtube_dl/extractor/vporn.py
@@ -64,29 +64,29 @@ class VpornIE(InfoExtractor):
         title = self._html_search_regex(
             r'videoname\s*=\s*\'([^\']+)\'', webpage, 'title').strip()
         description = self._html_search_regex(
-            r'<div class="description_txt">(.*?)</div>', webpage, 'description', fatal=False)
+            r'class="(?:descr|description_txt)">(.*?)</div>',
+            webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
             r'flashvars\.imageUrl\s*=\s*"([^"]+)"', webpage, 'description', fatal=False, default=None)
         if thumbnail:
             thumbnail = 'http://www.vporn.com' + thumbnail
 
         uploader = self._html_search_regex(
-            r'(?s)UPLOADED BY.*?<a href="/user/[^"]+">([^<]+)</a>',
+            r'(?s)Uploaded by:.*?<a href="/user/[^"]+">([^<]+)</a>',
             webpage, 'uploader', fatal=False)
 
         categories = re.findall(r'<a href="/cat/[^"]+">([^<]+)</a>', webpage)
 
         duration = parse_duration(self._search_regex(
-            r'duration (\d+ min \d+ sec)', webpage, 'duration', fatal=False))
+            r'Runtime:\s*</span>\s*(\d+ min \d+ sec)',
+            webpage, 'duration', fatal=False))
 
-        view_count = str_to_int(self._html_search_regex(
-            r'<span>([\d,\.]+) VIEWS</span>', webpage, 'view count', fatal=False))
-        like_count = str_to_int(self._html_search_regex(
-            r'<span id="like" class="n">([\d,\.]+)</span>', webpage, 'like count', fatal=False))
-        dislike_count = str_to_int(self._html_search_regex(
-            r'<span id="dislike" class="n">([\d,\.]+)</span>', webpage, 'dislike count', fatal=False))
+        view_count = str_to_int(self._search_regex(
+            r'class="views">([\d,\.]+) [Vv]iews<',
+            webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'<h4>Comments \(<b>([\d,\.]+)</b>\)</h4>', webpage, 'comment count', fatal=False))
+            r"'Comments \(([\d,\.]+)\)'",
+            webpage, 'comment count', default=None))
 
         formats = []
 
@@ -117,8 +117,6 @@ class VpornIE(InfoExtractor):
             'categories': categories,
             'duration': duration,
             'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
             'comment_count': comment_count,
             'age_limit': 18,
             'formats': formats,

From 482a1258de6af0a15b6e7859d244f9125cadef47 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 22:58:03 +0800
Subject: [PATCH 0430/1515] [VeeHD] Replace the third test case due to
 copyright issues

---
 youtube_dl/extractor/veehd.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 7fdeb784d..346edf485 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -39,13 +39,13 @@ class VeeHDIE(InfoExtractor):
             'uploader_id': 'spotted',
         }
     }, {
-        'url': 'http://veehd.com/video/4665804_Tell-No-One-Ne-le-dis-a-personne-2006-French-EngSoftSubs-Re-Up',
+        'url': 'http://veehd.com/video/2046729_2012-2009-DivX-Trailer',
         'info_dict': {
-            'id': '4665804',
+            'id': '2046729',
             'ext': 'avi',
-            'title': 'Tell No One (Ne le dis a personne) 2006 French(EngSoftSubs) Re-Up',
-            'description': 'md5:d660cca685549776f37165e9a10b60ba',
-            'uploader_id': 'belial2549',
+            'title': '2012 (2009) DivX Trailer',
+            'description': 'md5:75435ee95255e6a9838ac6f6f3a2396b',
+            'uploader_id': 'Movie_Trailers',
         }
     }]
 

From 7a03280df4555998fc99399907062b62383db2c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 21:31:38 +0600
Subject: [PATCH 0431/1515] [vporn] More metadata extraction fixes and tests
 update (#5560)

---
 youtube_dl/extractor/vporn.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vporn.py b/youtube_dl/extractor/vporn.py
index 04ed0b381..92c90e517 100644
--- a/youtube_dl/extractor/vporn.py
+++ b/youtube_dl/extractor/vporn.py
@@ -27,9 +27,6 @@ class VpornIE(InfoExtractor):
                 'duration': 393,
                 'age_limit': 18,
                 'view_count': int,
-                'like_count': int,
-                'dislike_count': int,
-                'comment_count': int,
             }
         },
         {
@@ -47,9 +44,6 @@ class VpornIE(InfoExtractor):
                 'duration': 588,
                 'age_limit': 18,
                 'view_count': int,
-                'like_count': int,
-                'dislike_count': int,
-                'comment_count': int,
             }
         },
     ]
@@ -72,10 +66,10 @@ class VpornIE(InfoExtractor):
             thumbnail = 'http://www.vporn.com' + thumbnail
 
         uploader = self._html_search_regex(
-            r'(?s)Uploaded by:.*?<a href="/user/[^"]+">([^<]+)</a>',
+            r'(?s)Uploaded by:.*?<a href="/user/[^"]+"[^>]*>(.+?)</a>',
             webpage, 'uploader', fatal=False)
 
-        categories = re.findall(r'<a href="/cat/[^"]+">([^<]+)</a>', webpage)
+        categories = re.findall(r'<a href="/cat/[^"]+"[^>]*>([^<]+)</a>', webpage)
 
         duration = parse_duration(self._search_regex(
             r'Runtime:\s*</span>\s*(\d+ min \d+ sec)',

From 4070b458ece46a29dad9be2312a7daa48bb2f1d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 23:55:05 +0600
Subject: [PATCH 0432/1515] [YoutubeDL] Do not write requested info in info
 JSON file (Closes #5562, closes #5564)

---
 youtube_dl/YoutubeDL.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9d4a2dce8..e747c6892 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1337,8 +1337,11 @@ class YoutubeDL(object):
                 self.to_screen('[info] Video description metadata is already present')
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
+                filtered_info_dict = dict(
+                    (k, v) for k, v in info_dict.items()
+                    if not k in ['requested_formats', 'requested_subtitles'])
                 try:
-                    write_json_file(info_dict, infofn)
+                    write_json_file(filtered_info_dict, infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return

From df8301fef55f9144f06337c10b8570b6560caa24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 30 Apr 2015 20:18:42 +0200
Subject: [PATCH 0433/1515] [YoutubeDL] pep8: use 'k not in' instead of 'not k
 in'

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e747c6892..584dbf8a6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1339,7 +1339,7 @@ class YoutubeDL(object):
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 filtered_info_dict = dict(
                     (k, v) for k, v in info_dict.items()
-                    if not k in ['requested_formats', 'requested_subtitles'])
+                    if k not in ['requested_formats', 'requested_subtitles'])
                 try:
                     write_json_file(filtered_info_dict, infofn)
                 except (OSError, IOError):

From 67fc8ecd53b4ffe5375a741bf0b1282f7a44587d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 30 Apr 2015 21:26:55 +0300
Subject: [PATCH 0434/1515] [dreisat] Extend _VALID_URL (Closes #5548)

---
 youtube_dl/extractor/dreisat.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 05bb22ddf..8ac8587be 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -11,19 +11,25 @@ from ..utils import (
 
 class DreiSatIE(InfoExtractor):
     IE_NAME = '3sat'
-    _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
-    _TEST = {
-        'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
-        'md5': 'be37228896d30a88f315b638900a026e',
-        'info_dict': {
-            'id': '45918',
-            'ext': 'mp4',
-            'title': 'Waidmannsheil',
-            'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
-            'uploader': '3sat',
-            'upload_date': '20140913'
-        }
-    }
+    _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
+    _TESTS = [
+        {
+            'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
+            'md5': 'be37228896d30a88f315b638900a026e',
+            'info_dict': {
+                'id': '45918',
+                'ext': 'mp4',
+                'title': 'Waidmannsheil',
+                'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
+                'uploader': '3sat',
+                'upload_date': '20140913'
+            }
+        },
+        {
+            'url': 'http://www.3sat.de/mediathek/mediathek.php?mode=play&obj=51066',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From cb202fd28635bf82836a025c631339665ba610af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 00:44:34 +0600
Subject: [PATCH 0435/1515] [YoutubeDL] Filter requested info fields on
 `--load-info` as well

In order to properly handle JSON info files generated by youtube-dl versions prior to 4070b458ece46a29dad9be2312a7daa48bb2f1d7
---
 youtube_dl/YoutubeDL.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 584dbf8a6..55b429f31 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1337,11 +1337,8 @@ class YoutubeDL(object):
                 self.to_screen('[info] Video description metadata is already present')
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
-                filtered_info_dict = dict(
-                    (k, v) for k, v in info_dict.items()
-                    if k not in ['requested_formats', 'requested_subtitles'])
                 try:
-                    write_json_file(filtered_info_dict, infofn)
+                    write_json_file(self.filter_requested_info(info_dict), infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
@@ -1491,7 +1488,7 @@ class YoutubeDL(object):
                 [info_filename], mode='r',
                 openhook=fileinput.hook_encoded('utf-8'))) as f:
             # FileInput doesn't have a read method, we can't call json.load
-            info = json.loads('\n'.join(f))
+            info = self.filter_requested_info(json.loads('\n'.join(f)))
         try:
             self.process_ie_result(info, download=True)
         except DownloadError:
@@ -1503,6 +1500,12 @@ class YoutubeDL(object):
                 raise
         return self._download_retcode
 
+    @staticmethod
+    def filter_requested_info(info_dict):
+        return dict(
+            (k, v) for k, v in info_dict.items()
+            if k not in ['requested_formats', 'requested_subtitles'])
+
     def post_process(self, filename, ie_info):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)

From 6a8422b942f5140238106c43e27d869d70126446 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 02:49:06 +0600
Subject: [PATCH 0436/1515] [foxsports] Add extractor (Closes #5517)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/foxsports.py | 32 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/foxsports.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 641c45f43..fced42bd9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -161,6 +161,7 @@ from .footyroom import FootyRoomIE
 from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
+from .foxsports import FoxSportsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
new file mode 100644
index 000000000..363866b64
--- /dev/null
+++ b/youtube_dl/extractor/foxsports.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class FoxSportsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/video\?vid=(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.foxsports.com/video?vid=432609859715',
+        'info_dict': {
+            'id': 'gA0bHB3Ladz3',
+            'ext': 'flv',
+            'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
+            'description': 'Courtney Lee talks about Memphis being focused.',
+        },
+        'add_ie': ['ThePlatform'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        config = self._parse_json(
+            self._search_regex(
+                r"data-player-config='([^']+)'", webpage, 'data player config'),
+            video_id)
+
+        return self.url_result(smuggle_url(
+            config['releaseURL'] + '&manifest=f4m', {'force_smil_url': True}))

From 1dbd717eb49d075fa1efabc674e8074fd165eb0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 02:51:55 +0600
Subject: [PATCH 0437/1515] [theplaform] Fix FutureWarning

---
 youtube_dl/extractor/theplatform.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 6a006b2d2..92731ad3d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -129,7 +129,9 @@ class ThePlatformIE(InfoExtractor):
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))
 
-        f4m_node = body.find(_x('smil:seq//smil:video')) or body.find(_x('smil:seq/smil:video'))
+        f4m_node = body.find(_x('smil:seq//smil:video'))
+        if f4m_node is None:
+            f4m_node = body.find(_x('smil:seq/smil:video'))
         if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
             f4m_url = f4m_node.attrib['src']
             if 'manifest.f4m?' not in f4m_url:
@@ -142,7 +144,9 @@ class ThePlatformIE(InfoExtractor):
             formats = []
             switch = body.find(_x('smil:switch'))
             if switch is None:
-                switch = body.find(_x('smil:par//smil:switch')) or body.find(_x('smil:par/smil:switch'))
+                switch = body.find(_x('smil:par//smil:switch'))
+            if switch is None:
+                switch = body.find(_x('smil:par/smil:switch'))
             if switch is None:
                 switch = body.find(_x('smil:par'))
             if switch is not None:
@@ -163,7 +167,9 @@ class ThePlatformIE(InfoExtractor):
                         'vbr': vbr,
                     })
             else:
-                switch = body.find(_x('smil:seq//smil:switch')) or body.find(_x('smil:seq/smil:switch'))
+                switch = body.find(_x('smil:seq//smil:switch'))
+                if switch is None:
+                    switch = body.find(_x('smil:seq/smil:switch'))
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
                     vbr = int_or_none(attr.get('system-bitrate'), 1000)

From 8683b4d8d91a7c6b72ca4a12bf6b538cbb4b2a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 03:59:13 +0600
Subject: [PATCH 0438/1515] [bbccouk] Improve extraction (Closes #5530)

---
 youtube_dl/extractor/bbccouk.py | 35 ++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index abc34a576..22c2843be 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
 from ..compat import compat_HTTPError
 
 
@@ -326,16 +329,29 @@ class BBCCoUkIE(InfoExtractor):
 
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
-        programme_id = self._search_regex(
-            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        programme_id = None
+
+        tviplayer = self._search_regex(
+            r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
+            webpage, 'player', default=None)
+
+        if tviplayer:
+            player = self._parse_json(tviplayer, group_id).get('player', {})
+            duration = int_or_none(player.get('duration'))
+            programme_id = player.get('vpid')
+
+        if not programme_id:
+            programme_id = self._search_regex(
+                r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+
         if programme_id:
-            player = self._download_json(
-                'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,
-                group_id)['jsConf']['player']
-            title = player['title']
-            description = player['subtitle']
-            duration = player['duration']
             formats, subtitles = self._download_media_selector(programme_id)
+            title = self._og_search_title(webpage)
+            description = self._search_regex(
+                r'<p class="medium-description">([^<]+)</p>',
+                webpage, 'description', fatal=False)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 
@@ -345,6 +361,7 @@ class BBCCoUkIE(InfoExtractor):
             'id': programme_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles,

From e68ae99a417f39db269dcffb5011cfcc8341552d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 04:02:56 +0600
Subject: [PATCH 0439/1515] [bbccouk] Add test for #5530

---
 youtube_dl/extractor/bbccouk.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 22c2843be..dbfbbb5ca 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -115,6 +115,20 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             }
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls',
+            'info_dict': {
+                'id': 'p02n76xf',
+                'ext': 'flv',
+                'title': 'Natural World, 2015-2016: 2. Super Powered Owls',
+                'description': 'md5:e4db5c937d0e95a7c6b5e654d429183d',
+                'duration': 3540,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'geolocation',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,

From 650cfd0cb0e330c8e6b1a5cc43a5a20d54b4714c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 04:07:30 +0600
Subject: [PATCH 0440/1515] [bbccouk] Mute thumbnail

---
 youtube_dl/extractor/bbccouk.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index dbfbbb5ca..249bc6bbd 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -343,8 +343,6 @@ class BBCCoUkIE(InfoExtractor):
 
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
-        thumbnail = self._og_search_thumbnail(webpage)
-
         programme_id = None
 
         tviplayer = self._search_regex(
@@ -375,7 +373,7 @@ class BBCCoUkIE(InfoExtractor):
             'id': programme_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles,

From 861e65eb0573c824cf82e1f31b7169df2efa74ab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 1 May 2015 12:31:31 +0800
Subject: [PATCH 0441/1515] [yahoo] Extend _VALID_URL

---
 youtube_dl/extractor/yahoo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b777159c5..bf4e659ac 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -22,7 +22,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
+    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -140,12 +140,15 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
                 'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
             }
+        }, {
+            'url': 'https://tw.news.yahoo.com/-100120367.html',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or self._match_id(url)
         page_id = mobj.group('id')
         url = mobj.group('url')
         host = mobj.group('host')

From 083c1bb960715031aed63dfb834a5bdc5ac6ff9a Mon Sep 17 00:00:00 2001
From: Nikoli <nikoli@gmx.us>
Date: Wed, 15 Apr 2015 20:27:40 +0300
Subject: [PATCH 0442/1515] Add ability to embed subtitles in mkv files (closes
 #5434)

---
 README.md                          | 2 +-
 youtube_dl/options.py              | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 7 ++++---
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index e1f30ca47..3432546fc 100644
--- a/README.md
+++ b/README.md
@@ -216,7 +216,7 @@ which means you can modify it, redistribute it or use it however you like.
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     Embed subtitles in the video (only for mp4 videos)
+    --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 4c9d39d9a..d0aa8296d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -698,7 +698,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,
-        help='Embed subtitles in the video (only for mp4 videos)')
+        help='Embed subtitles in the video (only for mkv and mp4 videos)')
     postproc.add_option(
         '--embed-thumbnail',
         action='store_true', dest='embedthumbnail', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1765f4969..214de39f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -501,8 +501,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         return cls._lang_map.get(code[:2])
 
     def run(self, information):
-        if information['ext'] != 'mp4':
-            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
+        if information['ext'] not in ['mp4', 'mkv']:
+            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
             return [], information
         subtitles = information.get('requested_subtitles')
         if not subtitles:
@@ -520,8 +520,9 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',
-            '-c:s', 'mov_text',
         ]
+        if information['ext'] == 'mp4':
+            opts += ['-c:s', 'mov_text']
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1)])
             lang_code = self._conver_lang_code(lang)

From 5890eef6b021845cb68882107364f1b04d773913 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 17:43:06 +0600
Subject: [PATCH 0443/1515] [pbs] Add support for HD (Closes #3564, closes
 #5390)

---
 youtube_dl/extractor/pbs.py | 56 ++++++++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index afce732e1..761bd6d8d 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -5,6 +5,8 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    determine_ext,
+    int_or_none,
     unified_strdate,
     US_RATINGS,
 )
@@ -149,21 +151,44 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
-        info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
-        info = self._download_json(info_url, display_id)
+        info = self._download_json(
+            'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+            display_id)
 
-        redirect_url = info['alternate_encoding']['url']
-        redirect_info = self._download_json(
-            redirect_url + '?format=json', display_id,
-            'Downloading video url info')
-        if redirect_info['status'] == 'error':
-            if redirect_info['http_code'] == 403:
-                message = (
-                    'The video is not available in your region due to '
-                    'right restrictions')
+        formats = []
+        for encoding_name in ('recommended_encoding', 'alternate_encoding'):
+            redirect = info.get(encoding_name)
+            if not redirect:
+                continue
+            redirect_url = redirect.get('url')
+            if not redirect_url:
+                continue
+
+            redirect_info = self._download_json(
+                redirect_url + '?format=json', display_id,
+                'Downloading %s video url info' % encoding_name)
+
+            if redirect_info['status'] == 'error':
+                if redirect_info['http_code'] == 403:
+                    message = (
+                        'The video is not available in your region due to '
+                        'right restrictions')
+                else:
+                    message = redirect_info['message']
+                raise ExtractorError(message, expected=True)
+
+            format_url = redirect_info.get('url')
+            if not format_url:
+                continue
+
+            if determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, display_id, 'mp4', preference=1, m3u8_id='hls'))
             else:
-                message = redirect_info['message']
-            raise ExtractorError(message, expected=True)
+                formats.append({
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
 
         rating_str = info.get('rating')
         if rating_str is not None:
@@ -174,11 +199,10 @@ class PBSIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'url': redirect_info['url'],
-            'ext': 'mp4',
             'description': info['program'].get('description'),
             'thumbnail': info.get('image_url'),
-            'duration': info.get('duration'),
+            'duration': int_or_none(info.get('duration')),
             'age_limit': age_limit,
             'upload_date': upload_date,
+            'formats': formats,
         }

From 8e3df9dfeef8503e9a8c01fcf42008d376d8d64d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 00:08:38 +0800
Subject: [PATCH 0444/1515] [viki] Fix extractor and add a global availble test
 case

---
 youtube_dl/extractor/viki.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 6816dacb6..957e3c01e 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -2,7 +2,10 @@ from __future__ import unicode_literals
 
 import re
 
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
     unescapeHTML,
@@ -15,8 +18,11 @@ from .common import InfoExtractor
 class VikiIE(InfoExtractor):
     IE_NAME = 'viki'
 
+    # iPad2
+    _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
+
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
             'id': '1023585v',
@@ -28,7 +34,17 @@ class VikiIE(InfoExtractor):
             'age_limit': 13,
         },
         'skip': 'Blocked in the US',
-    }
+    }, {
+        'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
+        'md5': 'ca6493e6f0a6ec07da9aa8d6304b4b2c',
+        'info_dict': {
+            'id': '1067139v',
+            'ext': 'mp4',
+            'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
+            'upload_date': '20150430',
+            'title': '\'The Avengers: Age of Ultron\' Press Conference',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -50,9 +66,11 @@ class VikiIE(InfoExtractor):
             'rating information', default='').strip()
         age_limit = US_RATINGS.get(rating_str)
 
-        info_url = 'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id
+        req = compat_urllib_request.Request(
+            'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id)
+        req.add_header('User-Agent', self._USER_AGENT)
         info_webpage = self._download_webpage(
-            info_url, video_id, note='Downloading info page')
+            req, video_id, note='Downloading info page')
         if re.match(r'\s*<div\s+class="video-error', info_webpage):
             raise ExtractorError(
                 'Video %s is blocked from your location.' % video_id,

From 89966a5aeacb70cc19b0a87a0514be824d2409af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 00:32:46 +0800
Subject: [PATCH 0445/1515] [viki] Enhance error message handling (#3774)

---
 youtube_dl/extractor/viki.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 957e3c01e..0fc1ceb19 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -11,6 +11,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     US_RATINGS,
+    clean_html,
 )
 from .common import InfoExtractor
 
@@ -71,10 +72,15 @@ class VikiIE(InfoExtractor):
         req.add_header('User-Agent', self._USER_AGENT)
         info_webpage = self._download_webpage(
             req, video_id, note='Downloading info page')
-        if re.match(r'\s*<div\s+class="video-error', info_webpage):
-            raise ExtractorError(
-                'Video %s is blocked from your location.' % video_id,
-                expected=True)
+        err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
+        if err_msg:
+            err_msg = clean_html(err_msg)
+            if 'not available in your region' in err_msg:
+                raise ExtractorError(
+                    'Video %s is blocked from your location.' % video_id,
+                    expected=True)
+            else:
+                raise ExtractorError('Viki said: ' + err_msg)
         video_url = self._html_search_regex(
             r'<source[^>]+src="([^"]+)"', info_webpage, 'video URL')
 

From d948e09b6174179adb43ac10cdb95dffeb4854dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 01:19:06 +0800
Subject: [PATCH 0446/1515] [viki] Extract m3u8 videos (#4855)

---
 youtube_dl/extractor/viki.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0fc1ceb19..f85e43042 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -12,6 +12,8 @@ from ..utils import (
     unified_strdate,
     US_RATINGS,
     clean_html,
+    determine_ext,
+    mimetype2ext,
 )
 from .common import InfoExtractor
 
@@ -45,6 +47,19 @@ class VikiIE(InfoExtractor):
             'upload_date': '20150430',
             'title': '\'The Avengers: Age of Ultron\' Press Conference',
         }
+    }, {
+        'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
+        'info_dict': {
+            'id': '1048879v',
+            'ext': 'mp4',
+            'upload_date': '20140820',
+            'description': 'md5:54ff56d51bdfc7a30441ec967394e91c',
+            'title': 'Ankhon Dekhi',
+        },
+        'params': {
+            # requires ffmpeg
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
@@ -81,8 +96,21 @@ class VikiIE(InfoExtractor):
                     expected=True)
             else:
                 raise ExtractorError('Viki said: ' + err_msg)
-        video_url = self._html_search_regex(
-            r'<source[^>]+src="([^"]+)"', info_webpage, 'video URL')
+        mobj = re.search(
+            r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
+        if not mobj:
+            raise ExtractorError('Unable to find video URL')
+        video_url = unescapeHTML(mobj.group('url'))
+        video_ext = mimetype2ext(mobj.group('mime_type'))
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                video_url, video_id, ext=video_ext)
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': video_ext,
+            }]
 
         upload_date_str = self._html_search_regex(
             r'"created_at":"([^"]+)"', info_webpage, 'upload date')
@@ -98,7 +126,7 @@ class VikiIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'description': description,
             'thumbnail': thumbnail,
             'age_limit': age_limit,

From 2eb0192155d53d70ae89ab8553ead49feb860ea4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 01:35:46 +0800
Subject: [PATCH 0447/1515] [viki] Remove clean_html call

---
 youtube_dl/extractor/viki.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index f85e43042..cf6af1e5c 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -11,7 +11,6 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     US_RATINGS,
-    clean_html,
     determine_ext,
     mimetype2ext,
 )
@@ -89,7 +88,6 @@ class VikiIE(InfoExtractor):
             req, video_id, note='Downloading info page')
         err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
         if err_msg:
-            err_msg = clean_html(err_msg)
             if 'not available in your region' in err_msg:
                 raise ExtractorError(
                     'Video %s is blocked from your location.' % video_id,

From c938c35f957ea069eed824131ca908608853abe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 07:18:22 +0600
Subject: [PATCH 0448/1515] [iconosquare] Fix extraction

---
 youtube_dl/extractor/iconosquare.py | 57 ++++++++++++++++++++++++-----
 1 file changed, 48 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 370e86e5a..70e4c0d41 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -1,36 +1,75 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class IconosquareIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?(?:iconosquare\.com|statigr\.am)/p/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:iconosquare\.com|statigr\.am)/p/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://statigr.am/p/522207370455279102_24101272',
         'md5': '6eb93b882a3ded7c378ee1d6884b1814',
         'info_dict': {
             'id': '522207370455279102_24101272',
             'ext': 'mp4',
-            'uploader_id': 'aguynamedpatrick',
-            'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)',
+            'title': 'Instagram media by @aguynamedpatrick (Patrick Janelle)',
             'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
+            'timestamp': 1376471991,
+            'upload_date': '20130814',
+            'uploader': 'aguynamedpatrick',
+            'uploader_id': '24101272',
+            'comment_count': int,
+            'like_count': int,
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
+
+        media = self._parse_json(
+            self._search_regex(
+                r'window\.media\s*=\s*({.+?});\n', webpage, 'media'),
+            video_id)
+
+        formats = [{
+            'url': f['url'],
+            'format_id': format_id,
+            'width': int_or_none(f.get('width')),
+            'height': int_or_none(f.get('height'))
+        } for format_id, f in media['videos'].items()]
+        self._sort_formats(formats)
+
         title = self._html_search_regex(
             r'<title>(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)</title>',
             webpage, 'title')
-        uploader_id = self._html_search_regex(
-            r'@([^ ]+)', title, 'uploader name', fatal=False)
+
+        timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))
+        description = media.get('caption', {}).get('text')
+
+        uploader = media.get('user', {}).get('username')
+        uploader_id = media.get('user', {}).get('id')
+
+        comment_count = int_or_none(media.get('comments', {}).get('count'))
+        like_count = int_or_none(media.get('likes', {}).get('count'))
+
+        thumbnails = [{
+            'url': t['url'],
+            'id': thumbnail_id,
+            'width': int_or_none(t.get('width')),
+            'height': int_or_none(t.get('height'))
+        } for thumbnail_id, t in media.get('images', {}).items()]
 
         return {
             'id': video_id,
-            'url': self._og_search_video_url(webpage),
             'title': title,
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader_id': uploader_id
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'formats': formats,
         }

From dd8920653c7249545bbcf5a690ffca702c9f29f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 21:45:25 +0800
Subject: [PATCH 0449/1515] [Grooveshark] Remove the extractor

grooveshark.com was shut down on 2015/04/30
---
 youtube_dl/extractor/__init__.py    |   1 -
 youtube_dl/extractor/grooveshark.py | 191 ----------------------------
 2 files changed, 192 deletions(-)
 delete mode 100644 youtube_dl/extractor/grooveshark.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fced42bd9..43f3e6808 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -199,7 +199,6 @@ from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
-from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
 from .hearthisat import HearThisAtIE
diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
deleted file mode 100644
index 36ad4915c..000000000
--- a/youtube_dl/extractor/grooveshark.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import time
-import math
-import os.path
-import re
-
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_html_parser,
-    compat_urllib_parse,
-    compat_urllib_request,
-    compat_urlparse,
-)
-from ..utils import ExtractorError
-
-
-class GroovesharkHtmlParser(compat_html_parser.HTMLParser):
-    def __init__(self):
-        self._current_object = None
-        self.objects = []
-        compat_html_parser.HTMLParser.__init__(self)
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict((k, v) for k, v in attrs)
-        if tag == 'object':
-            self._current_object = {'attrs': attrs, 'params': []}
-        elif tag == 'param':
-            self._current_object['params'].append(attrs)
-
-    def handle_endtag(self, tag):
-        if tag == 'object':
-            self.objects.append(self._current_object)
-            self._current_object = None
-
-    @classmethod
-    def extract_object_tags(cls, html):
-        p = cls()
-        p.feed(html)
-        p.close()
-        return p.objects
-
-
-class GroovesharkIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?grooveshark\.com/#!/s/([^/]+)/([^/]+)'
-    _TEST = {
-        'url': 'http://grooveshark.com/#!/s/Jolene+Tenth+Key+Remix+Ft+Will+Sessions/6SS1DW?src=5',
-        'md5': '7ecf8aefa59d6b2098517e1baa530023',
-        'info_dict': {
-            'id': '6SS1DW',
-            'title': 'Jolene (Tenth Key Remix ft. Will Sessions)',
-            'ext': 'mp3',
-            'duration': 227,
-        }
-    }
-
-    do_playerpage_request = True
-    do_bootstrap_request = True
-
-    def _parse_target(self, target):
-        uri = compat_urlparse.urlparse(target)
-        hash = uri.fragment[1:].split('?')[0]
-        token = os.path.basename(hash.rstrip('/'))
-        return (uri, hash, token)
-
-    def _build_bootstrap_url(self, target):
-        (uri, hash, token) = self._parse_target(target)
-        query = 'getCommunicationToken=1&hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts)
-        return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token)
-
-    def _build_meta_url(self, target):
-        (uri, hash, token) = self._parse_target(target)
-        query = 'hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts)
-        return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token)
-
-    def _build_stream_url(self, meta):
-        return compat_urlparse.urlunparse(('http', meta['streamKey']['ip'], '/stream.php', None, None, None))
-
-    def _build_swf_referer(self, target, obj):
-        (uri, _, _) = self._parse_target(target)
-        return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None))
-
-    def _transform_bootstrap(self, js):
-        return re.split('(?m)^\s*try\s*\{', js)[0] \
-                 .split(' = ', 1)[1].strip().rstrip(';')
-
-    def _transform_meta(self, js):
-        return js.split('\n')[0].split('=')[1].rstrip(';')
-
-    def _get_meta(self, target):
-        (meta_url, token) = self._build_meta_url(target)
-        self.to_screen('Metadata URL: %s' % meta_url)
-
-        headers = {'Referer': compat_urlparse.urldefrag(target)[0]}
-        req = compat_urllib_request.Request(meta_url, headers=headers)
-        res = self._download_json(req, token,
-                                  transform_source=self._transform_meta)
-
-        if 'getStreamKeyWithSong' not in res:
-            raise ExtractorError(
-                'Metadata not found. URL may be malformed, or Grooveshark API may have changed.')
-
-        if res['getStreamKeyWithSong'] is None:
-            raise ExtractorError(
-                'Metadata download failed, probably due to Grooveshark anti-abuse throttling. Wait at least an hour before retrying from this IP.',
-                expected=True)
-
-        return res['getStreamKeyWithSong']
-
-    def _get_bootstrap(self, target):
-        (bootstrap_url, token) = self._build_bootstrap_url(target)
-
-        headers = {'Referer': compat_urlparse.urldefrag(target)[0]}
-        req = compat_urllib_request.Request(bootstrap_url, headers=headers)
-        res = self._download_json(req, token, fatal=False,
-                                  note='Downloading player bootstrap data',
-                                  errnote='Unable to download player bootstrap data',
-                                  transform_source=self._transform_bootstrap)
-        return res
-
-    def _get_playerpage(self, target):
-        (_, _, token) = self._parse_target(target)
-
-        webpage = self._download_webpage(
-            target, token,
-            note='Downloading player page',
-            errnote='Unable to download player page',
-            fatal=False)
-
-        if webpage is not None:
-            # Search (for example German) error message
-            error_msg = self._html_search_regex(
-                r'<div id="content">\s*<h2>(.*?)</h2>', webpage,
-                'error message', default=None)
-            if error_msg is not None:
-                error_msg = error_msg.replace('\n', ' ')
-                raise ExtractorError('Grooveshark said: %s' % error_msg)
-
-        if webpage is not None:
-            o = GroovesharkHtmlParser.extract_object_tags(webpage)
-            return webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed']
-
-        return webpage, None
-
-    def _real_initialize(self):
-        self.ts = int(time.time() * 1000)  # timestamp in millis
-
-    def _real_extract(self, url):
-        (target_uri, _, token) = self._parse_target(url)
-
-        # 1. Fill cookiejar by making a request to the player page
-        swf_referer = None
-        if self.do_playerpage_request:
-            (_, player_objs) = self._get_playerpage(url)
-            if player_objs:
-                swf_referer = self._build_swf_referer(url, player_objs[0])
-                self.to_screen('SWF Referer: %s' % swf_referer)
-
-        # 2. Ask preload.php for swf bootstrap data to better mimic webapp
-        if self.do_bootstrap_request:
-            bootstrap = self._get_bootstrap(url)
-            self.to_screen('CommunicationToken: %s' % bootstrap['getCommunicationToken'])
-
-        # 3. Ask preload.php for track metadata.
-        meta = self._get_meta(url)
-
-        # 4. Construct stream request for track.
-        stream_url = self._build_stream_url(meta)
-        duration = int(math.ceil(float(meta['streamKey']['uSecs']) / 1000000))
-        post_dict = {'streamKey': meta['streamKey']['streamKey']}
-        post_data = compat_urllib_parse.urlencode(post_dict).encode('utf-8')
-        headers = {
-            'Content-Length': len(post_data),
-            'Content-Type': 'application/x-www-form-urlencoded'
-        }
-        if swf_referer is not None:
-            headers['Referer'] = swf_referer
-
-        return {
-            'id': token,
-            'title': meta['song']['Name'],
-            'http_method': 'POST',
-            'url': stream_url,
-            'ext': 'mp3',
-            'format': 'mp3 audio',
-            'duration': duration,
-            'http_post_data': post_data,
-            'http_headers': headers,
-        }

From 2ddcd88129e633ead8c5faee6c930b66dddb1b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 May 2015 17:29:56 +0200
Subject: [PATCH 0450/1515] Remove code that was only used by the Grooveshark
 extractor

---
 youtube_dl/compat.py           | 6 ------
 youtube_dl/downloader/http.py  | 9 ++-------
 youtube_dl/extractor/common.py | 3 ---
 3 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 973bcd320..f9529210d 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -46,11 +46,6 @@ try:
 except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
-try:
-    import html.parser as compat_html_parser
-except ImportError:  # Python 2
-    import HTMLParser as compat_html_parser
-
 try:
     import http.client as compat_http_client
 except ImportError:  # Python 2
@@ -404,7 +399,6 @@ __all__ = [
     'compat_getenv',
     'compat_getpass',
     'compat_html_entities',
-    'compat_html_parser',
     'compat_http_client',
     'compat_http_server',
     'compat_kwargs',
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d136bebd1..b7f144af9 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -28,13 +28,8 @@ class HttpFD(FileDownloader):
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
-        data = info_dict.get('http_post_data')
-        http_method = info_dict.get('http_method')
-        basic_request = compat_urllib_request.Request(url, data, headers)
-        request = compat_urllib_request.Request(url, data, headers)
-        if http_method is not None:
-            basic_request.get_method = lambda: http_method
-            request.get_method = lambda: http_method
+        basic_request = compat_urllib_request.Request(url, None, headers)
+        request = compat_urllib_request.Request(url, None, headers)
 
         is_test = self.params.get('test', False)
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 48f58aae3..3ae5d5212 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -111,11 +111,8 @@ class InfoExtractor(object):
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
-                    * http_method  HTTP method to use for the download.
                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.
-                    * http_post_data  Additional data to send with a POST
-                                 request.
                     * stretched_ratio  If given and not 1, indicates that the
                                  video's pixels are not square.
                                  width : height ratio as float.

From 38c6902b9051fa7e1dbda79808a90dbf2aac37ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 22:52:21 +0600
Subject: [PATCH 0451/1515] [YoutubeDL] Ensure correct extension is always
 present for a merged file (Closes #5535)

---
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 55b429f31..eaa436bf9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1382,11 +1382,18 @@ class YoutubeDL(object):
                         # TODO: Check acodec/vcodec
                         return False
 
+                    filename_real_ext = os.path.splitext(filename)[1][1:]
+                    filename_wo_ext = (
+                        os.path.splitext(filename)[0]
+                        if filename_real_ext == info_dict['ext']
+                        else filename)
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
-                        filename = os.path.splitext(filename)[0] + '.mkv'
+                        info_dict['ext'] = 'mkv'
                         self.report_warning('You have requested formats incompatible for merge. '
                                             'The formats will be merged into mkv')
+                    # Ensure filename always has a correct extension for successful merge
+                    filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(
                             '[download] %s has already been downloaded and '

From 21f6330274c6f87c796bd9248ed82d2bc73de969 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 May 2015 00:51:24 +0800
Subject: [PATCH 0452/1515] [baiduvideo] Add new extractor (closes #4563)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/baidu.py    | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/baidu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 43f3e6808..41af925cc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -32,6 +32,7 @@ from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .azubu import AzubuIE
+from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
new file mode 100644
index 000000000..7f7a88ffe
--- /dev/null
+++ b/youtube_dl/extractor/baidu.py
@@ -0,0 +1,68 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+
+
+class BaiduVideoIE(InfoExtractor):
+    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+).htm'
+    _TESTS = [{
+        'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
+        'info_dict': {
+            'id': '1069',
+            'title': '中华小当家 TV版 (全52集)',
+            'description': 'md5:395a419e41215e531c857bb037bbaf80',
+        },
+        'playlist_count': 52,
+    }, {
+        'url': 'http://v.baidu.com/show/11595.htm?frp=bdbrand',
+        'info_dict': {
+            'id': '11595',
+            'title': 're:^奔跑吧兄弟',
+            'description': 'md5:1bf88bad6d850930f542d51547c089b8',
+        },
+        'playlist_mincount': 3,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('id')
+        category = category2 = mobj.group('type')
+        if category == 'show':
+            category2 = 'tvshow'
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist_title = self._html_search_regex(
+            r'title\s*:\s*(["\'])(?P<title>[^\']+)\1', webpage,
+            'playlist title', group='title')
+        playlist_description = self._html_search_regex(
+            r'<input[^>]+class="j-data-intro"[^>]+value="([^"]+)"/>', webpage,
+            playlist_id, 'playlist description')
+
+        site = self._html_search_regex(
+            r'filterSite\s*:\s*["\']([^"]*)["\']', webpage,
+            'primary provider site')
+        api_result = self._download_json(
+            'http://v.baidu.com/%s_intro/?dtype=%sPlayUrl&id=%s&site=%s' % (
+                category, category2, playlist_id, site),
+            playlist_id, 'Get playlist links')
+
+        entries = []
+        for episode in api_result[0]['episodes']:
+            episode_id = '%s_%s' % (playlist_id, episode['episode'])
+
+            redirect_page = self._download_webpage(
+                compat_urlparse.urljoin(url, episode['url']), episode_id,
+                note='Download Baidu redirect page')
+            real_url = self._html_search_regex(
+                r'location\.replace\("([^"]+)"\)', redirect_page, 'real URL')
+
+            entries.append(self.url_result(
+                real_url, video_title=episode['single_title']))
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_title, playlist_description)

From e65e4c88748f2d245aa683116a2c6de907186751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:06:01 +0600
Subject: [PATCH 0453/1515] [utils] Improve prepend_extension

Now `ext` is appended to filename if real extension != expected extension.
---
 youtube_dl/utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f07679c76..b3abfbc11 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1349,9 +1349,12 @@ def parse_duration(s):
     return res
 
 
-def prepend_extension(filename, ext):
+def prepend_extension(filename, ext, expected_real_ext=None):
     name, real_ext = os.path.splitext(filename)
-    return '{0}.{1}{2}'.format(name, ext, real_ext)
+    return (
+        '{0}.{1}{2}'.format(name, ext, real_ext)
+        if not expected_real_ext or real_ext[1:] == expected_real_ext
+        else '{0}.{1}'.format(filename, ext))
 
 
 def check_executable(exe, args=[]):

From a4bcaad77314c714d2304cb21ffdd87a9b84316b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:10:48 +0600
Subject: [PATCH 0454/1515] [test_utils] Add tests for prepend_extension

---
 test/test_utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 17017a8c0..a9464f2e4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -41,6 +41,7 @@ from youtube_dl.utils import (
     sanitize_filename,
     sanitize_path,
     sanitize_url_path_consecutive_slashes,
+    prepend_extension,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -193,6 +194,14 @@ class TestUtil(unittest.TestCase):
             sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
             'http://hostname/abc/')
 
+    def test_prepend_extension(self):
+        self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
+        self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
+        self.assertEqual(prepend_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp')
+        self.assertEqual(prepend_extension('abc', 'temp'), 'abc.temp')
+        self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
+        self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])

From 666a9a2b954bb6c75a5fcdb9fbb18842038c188a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:11:34 +0600
Subject: [PATCH 0455/1515] [YoutubeDL] Improve audio/video-only file naming

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eaa436bf9..06d04c8f0 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1403,7 +1403,7 @@ class YoutubeDL(object):
                             new_info = dict(info_dict)
                             new_info.update(f)
                             fname = self.prepare_filename(new_info)
-                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                            fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
                             downloaded.append(fname)
                             partial_success = dl(fname, new_info)
                             success = success and partial_success

From b3ed15b7604ce83e85b791ed329c5725a436b805 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:23:06 +0600
Subject: [PATCH 0456/1515] [utils] Add replace_extension

---
 test/test_utils.py  | 9 +++++++++
 youtube_dl/utils.py | 7 +++++++
 2 files changed, 16 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a9464f2e4..6906a65c2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -42,6 +42,7 @@ from youtube_dl.utils import (
     sanitize_path,
     sanitize_url_path_consecutive_slashes,
     prepend_extension,
+    replace_extension,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -202,6 +203,14 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
 
+    def test_replace_extension(self):
+        self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp')
+        self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp')
+        self.assertEqual(replace_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp')
+        self.assertEqual(replace_extension('abc', 'temp'), 'abc.temp')
+        self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
+        self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b3abfbc11..a5a5c317e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1357,6 +1357,13 @@ def prepend_extension(filename, ext, expected_real_ext=None):
         else '{0}.{1}'.format(filename, ext))
 
 
+def replace_extension(filename, ext, expected_real_ext=None):
+    name, real_ext = os.path.splitext(filename)
+    return '{0}.{1}'.format(
+        name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
+        ext)
+
+
 def check_executable(exe, args=[]):
     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
     args can be a list of arguments for a short output (like -version) """

From b29e0000e69458252f73ffe62a466da5d1449863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:23:44 +0600
Subject: [PATCH 0457/1515] [YoutubeDL] Improve JSON info file naming

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 06d04c8f0..0330b0b34 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -71,6 +71,7 @@ from .utils import (
     write_string,
     YoutubeDLHandler,
     prepend_extension,
+    replace_extension,
     args_to_str,
     age_restricted,
 )
@@ -1332,7 +1333,7 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeinfojson', False):
-            infofn = os.path.splitext(filename)[0] + '.info.json'
+            infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
                 self.to_screen('[info] Video description metadata is already present')
             else:

From 98727e123f6d4881920bcb9fa57ea7956afb9a5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:35:18 +0600
Subject: [PATCH 0458/1515] [YoutubeDL] Improve annotations file naming

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0330b0b34..27c1e726f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1286,7 +1286,7 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeannotations', False):
-            annofn = filename + '.annotations.xml'
+            annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
                 self.to_screen('[info] Video annotations are already present')
             else:

From 2699da8041cf10a636e0ac99efb91854b2c3aa58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:36:55 +0600
Subject: [PATCH 0459/1515] [YoutubeDL] Improve description file naming

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 27c1e726f..eb7470f72 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1271,7 +1271,7 @@ class YoutubeDL(object):
             return
 
         if self.params.get('writedescription', False):
-            descfn = filename + '.description'
+            descfn = replace_extension(filename, 'description', info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
             elif info_dict.get('description') is None:

From 0669c89c55a77eeda2b9767e096c465b12c67b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:38:30 +0600
Subject: [PATCH 0460/1515] [options] Clarify `--write-annotations` help

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d0aa8296d..22dbc3aec 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -637,7 +637,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
-        help='Write video annotations to a .annotation file')
+        help='Write video annotations to a .annotations.xml file')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',

From 0a64aa73552d3fc3ae6e24a56e2f8e78c960603c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 00:58:42 +0600
Subject: [PATCH 0461/1515] [vgtv] Fix _VALID_URL (Closes #5578)

---
 youtube_dl/extractor/vgtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 2f111bf7e..69dc9a759 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -8,7 +8,7 @@ from ..utils import float_or_none
 
 
 class VGTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/(?:.*)/(?P<id>[0-9]+)'
+    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/[^/]+/(?P<id>[0-9]+)'
     _TESTS = [
         {
             # streamType: vod

From 963aea5279691affca34d220452cecfd0e291a77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 07:45:15 +0600
Subject: [PATCH 0462/1515] [baiduvideo] Improve _VALID_URL

---
 youtube_dl/extractor/baidu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 7f7a88ffe..906895c1e 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -8,7 +8,7 @@ from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
-    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+).htm'
+    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
         'info_dict': {

From 8dab1e9072037daa9c6cab3da4a5dbd4daaae4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 09:56:03 +0600
Subject: [PATCH 0463/1515] [rutv] Recognize live streams (#5584)

---
 youtube_dl/extractor/rutv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index ef766237b..169f7c032 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -181,12 +181,15 @@ class RUTVIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        is_live = video_type == 'live'
+
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'description': description,
             'thumbnail': thumbnail,
             'view_count': view_count,
             'duration': duration,
             'formats': formats,
+            'is_live': is_live,
         }

From d0fd305023b37f7776485679a74e422eade26c13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 10:00:34 +0600
Subject: [PATCH 0464/1515] [rutv] Add test for #5584

---
 youtube_dl/extractor/rutv.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 169f7c032..1ec2c86e5 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -84,11 +84,20 @@ class RUTVIE(InfoExtractor):
                 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ',
                 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c',
             },
+            'skip': 'Translation has finished',
+        },
+        {
+            'url': 'http://live.russia.tv/index/index/channel_id/3',
+            'info_dict': {
+                'id': '21',
+                'ext': 'mp4',
+                'title': 're:^Россия 24. Прямой эфир [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'is_live': True,
+            },
             'params': {
-                # rtmp download
+                # m3u8 download
                 'skip_download': True,
             },
-            'skip': 'Translation has finished',
         },
     ]
 

From 5477ca82395545e577afb269b910336cd98de5b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 May 2015 16:59:14 +0200
Subject: [PATCH 0465/1515] [dailymotion] Use https urls

The video url still redirects to an http url, but it doesn't explicitly contain the video id.
---
 youtube_dl/extractor/dailymotion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7615ecd4b..aa595af20 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -85,7 +85,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'http://www.dailymotion.com/video/%s' % video_id
+        url = 'https://www.dailymotion.com/video/%s' % video_id
 
         # Retrieve video webpage to extract further information
         request = self._build_request(url)
@@ -110,7 +110,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         if mobj is not None:
             video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 
-        embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
+        embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
         embed_request = self._build_request(embed_url)
         embed_page = self._download_webpage(
             embed_request, video_id, 'Downloading embed page')

From 1748d67aea66b6d63bc203ada1036110e7fd2402 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 01:11:23 +0800
Subject: [PATCH 0466/1515] [lifenews] Fix view count and comment count

---
 youtube_dl/extractor/lifenews.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 1dfe7f77f..3c00ed333 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -47,9 +47,9 @@ class LifeNewsIE(InfoExtractor):
         description = self._og_search_description(webpage)
 
         view_count = self._html_search_regex(
-            r'<div class=\'views\'>(\d+)</div>', webpage, 'view count', fatal=False)
+            r'<div class=\'views\'>\s*(\d+)\s*</div>', webpage, 'view count', fatal=False)
         comment_count = self._html_search_regex(
-            r'<div class=\'comments\'>\s*<span class=\'counter\'>(\d+)</span>', webpage, 'comment count', fatal=False)
+            r'<div class=\'comments\'>\s*<span class=\'counter\'>\s*(\d+)\s*</span>', webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
             r'<time datetime=\'([^\']+)\'>', webpage, 'upload date', fatal=False)

From 848edeab898ee45e45a8fcdb355f35be8b73dd5d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 01:24:19 +0800
Subject: [PATCH 0467/1515] [lifenews] Detect <iframe> (fixes #5346)

---
 youtube_dl/extractor/lifenews.py | 47 ++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 3c00ed333..330138692 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -16,7 +16,7 @@ class LifeNewsIE(InfoExtractor):
     IE_DESC = 'LIFE | NEWS'
     _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?news/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://lifenews.ru/news/126342',
         'md5': 'e1b50a5c5fb98a6a544250f2e0db570a',
         'info_dict': {
@@ -27,7 +27,19 @@ class LifeNewsIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg',
             'upload_date': '20140130',
         }
-    }
+    }, {
+        # video in <iframe>
+        'url': 'http://lifenews.ru/news/152125',
+        'md5': '77d19a6f0886cd76bdbf44b4d971a273',
+        'info_dict': {
+            'id': '152125',
+            'ext': 'mp4',
+            'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
+            'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
+            'upload_date': '20150402',
+            'uploader': 'embed.life.ru',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -36,7 +48,9 @@ class LifeNewsIE(InfoExtractor):
         webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
 
         videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
-        if not videos:
+        iframe_link = self._html_search_regex(
+            '<iframe[^>]+src="([^"]+)', webpage, 'iframe link', default=None)
+        if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 
         title = self._og_search_title(webpage)
@@ -56,17 +70,32 @@ class LifeNewsIE(InfoExtractor):
         if upload_date is not None:
             upload_date = unified_strdate(upload_date)
 
+        common_info = {
+            'description': description,
+            'view_count': int_or_none(view_count),
+            'comment_count': int_or_none(comment_count),
+            'upload_date': upload_date,
+        }
+
         def make_entry(video_id, media, video_number=None):
-            return {
+            cur_info = dict(common_info)
+            cur_info.update({
                 'id': video_id,
                 'url': media[1],
                 'thumbnail': media[0],
                 'title': title if video_number is None else '%s-video%s' % (title, video_number),
-                'description': description,
-                'view_count': int_or_none(view_count),
-                'comment_count': int_or_none(comment_count),
-                'upload_date': upload_date,
-            }
+            })
+            return cur_info
+
+        if iframe_link:
+            cur_info = dict(common_info)
+            cur_info.update({
+                '_type': 'url_transparent',
+                'id': video_id,
+                'title': title,
+                'url': iframe_link,
+            })
+            return cur_info
 
         if len(videos) == 1:
             return make_entry(video_id, videos[0])

From 3e7202c1bca0618fef04c60a0f5603d50f09b76f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 01:59:26 +0800
Subject: [PATCH 0468/1515] [MLB] Extend _VALID_URL (#5443)

---
 youtube_dl/extractor/mlb.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index e369551c2..ee9ff73bf 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/(?:embed|m-internal-embed)\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -83,6 +83,11 @@ class MLBIE(InfoExtractor):
         {
             'url': 'http://m.cardinals.mlb.com/stl/video/v51175783/atlstl-piscotty-makes-great-sliding-catch-on-line/?partnerId=as_mlb_20150321_42500876&adbid=579409712979910656&adbpl=tw&adbpr=52847728',
             'only_matching': True,
+        },
+        {
+            # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
+            'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
+            'only_matching': True,
         }
     ]
 

From 8001607e903a5ccaff76908f23bece08818743c0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 02:20:07 +0800
Subject: [PATCH 0469/1515] [generic] Detect more MLB videos (fixes #5443)

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4946cc132..d09e85665 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -413,6 +413,19 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        # MLB articles
+        {
+            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
+            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
+            'info_dict': {
+                'id': '75609783',
+                'ext': 'mp4',
+                'title': 'Must C: Pillar climbs for catch',
+                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
+                'timestamp': 1429124820,
+                'upload_date': '20150415',
+            }
+        },
         # Wistia embed
         {
             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
@@ -1289,6 +1302,10 @@ class GenericIE(InfoExtractor):
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
+        if not mobj:
+            mobj = re.search(
+                r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
+                webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
 

From 957b794c2622fdc3696c245f1fa1eebfdb0f8bdf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 3 May 2015 22:31:39 +0200
Subject: [PATCH 0470/1515] release 2015.05.03

---
 README.md              | 2 +-
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3432546fc..9aeb114f3 100644
--- a/README.md
+++ b/README.md
@@ -133,7 +133,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-mtime                       Do not use the Last-modified header to set the file modification time
     --write-description              Write video description to a .description file
     --write-info-json                Write video metadata to a .info.json file
-    --write-annotations              Write video annotations to a .annotation file
+    --write-annotations              Write video annotations to a .annotations.xml file
     --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
     --cookies FILE                   File to read cookies from and dump cookie jar in
     --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d1d676cdd..6d2e496a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,6 +44,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
+ - **BaiduVideo**
  - **bambuser**
  - **bambuser:channel**
  - **Bandcamp**
@@ -155,6 +156,7 @@
  - **FootyRoom**
  - **Foxgay**
  - **FoxNews**
+ - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
  - **FranceInter**
@@ -184,7 +186,6 @@
  - **Golem**
  - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net
  - **Goshgay**
- - **Grooveshark**
  - **Groupon**
  - **Hark**
  - **HearThisAt**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b88ea85e8..6950afc47 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.28'
+__version__ = '2015.05.03'

From b9b3ab45ea95e4c48f3c87d3584f8180ff460be6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 19:09:18 +0800
Subject: [PATCH 0471/1515] [NBC] Enhance extraction of ThePlatform URL (fixes
 #5470)

---
 youtube_dl/extractor/nbc.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index ecd0ac8b1..6cbe03d0f 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -37,13 +37,26 @@ class NBCIE(InfoExtractor):
             },
             'skip': 'Only works from US',
         },
+        {
+            'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
+            'info_dict': {
+                'id': '8iUuyzWDdYUZ',
+                'ext': 'flv',
+                'title': 'Star Wars Teaser',
+                'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
+            },
+            'skip': 'Only works from US',
+        }
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         theplatform_url = self._search_regex(
-            '(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+            [
+                r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+                r'"embedURL"\s*:\s*"([^"]+)"'
+            ],
             webpage, 'theplatform url').replace('_no_endcard', '')
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url

From 71fa56b887b7c4d1a3ba16c7127736fcaec34b4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 18:59:22 +0600
Subject: [PATCH 0472/1515] [escapist] Fix formats extraction

---
 youtube_dl/extractor/escapist.py | 35 ++++++++++++++------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 8facf1185..802943dc2 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_request
 from ..utils import (
     determine_ext,
     clean_html,
-    qualities,
+    int_or_none,
 )
 
 
@@ -72,28 +72,23 @@ class EscapistIE(InfoExtractor):
         video_id = imsVideo['videoID']
         key = imsVideo['hash']
 
-        quality = qualities(['lq', 'hq', 'hd'])
+        config_req = compat_urllib_request.Request(
+            'http://www.escapistmagazine.com/videos/'
+            'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
+        config_req.add_header('Referer', url)
+        config = self._download_webpage(config_req, video_id, 'Downloading video config')
 
-        formats = []
-        for q in ['lq', 'hq', 'hd']:
-            config_req = compat_urllib_request.Request(
-                'http://www.escapistmagazine.com/videos/'
-                'vidconfig.php?videoID=%s&hash=%s&quality=%s' % (video_id, key, 'mp4_' + q))
-            config_req.add_header('Referer', url)
-            config = self._download_webpage(config_req, video_id, 'Downloading video config ' + q.upper())
+        data = json.loads(_decrypt_config(key, config))
 
-            data = json.loads(_decrypt_config(key, config))
+        title = clean_html(data['videoData']['title'])
+        duration = data['videoData']['duration'] / 1000
 
-            title = clean_html(data['videoData']['title'])
-            duration = data['videoData']['duration'] / 1000
-
-            for i, v in enumerate(data['files']['videos']):
-
-                formats.append({
-                    'url': v,
-                    'format_id': determine_ext(v) + '_' + q + str(i),
-                    'quality': quality(q),
-                })
+        formats = [{
+            'url': video['src'],
+            'format_id': '%s-%sp' % (determine_ext(video['src']), video['res']),
+            'height': int_or_none(video.get('res')),
+        } for video in data['files']['videos']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From cec04ef3a6cfd43036f7463dc1fa890af5b6ee11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:00:34 +0600
Subject: [PATCH 0473/1515] [escapist] Update tests' checksums

---
 youtube_dl/extractor/escapist.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 802943dc2..daf859385 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -39,7 +39,7 @@ class EscapistIE(InfoExtractor):
     _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
-        'md5': 'c6793dbda81388f4264c1ba18684a74d',
+        'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
         'info_dict': {
             'id': '6618',
             'ext': 'mp4',
@@ -50,7 +50,7 @@ class EscapistIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
-        'md5': 'cf8842a8a46444d241f9a9980d7874f2',
+        'md5': '9e8c437b0dbb0387d3bd3255ca77f6bf',
         'info_dict': {
             'id': '10044',
             'ext': 'mp4',

From 90b4b0eabeabbdad3bfece8254692d34a3dcba95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:01:08 +0600
Subject: [PATCH 0474/1515] [escapist] Improve _VALID_URL

---
 youtube_dl/extractor/escapist.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index daf859385..dadfaa6a5 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -36,7 +36,7 @@ def _decrypt_config(key, string):
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
+    _VALID_URL = r'https?://?(?:www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',

From 782e0568ef28ebd7d7a11273d0a5af065575379a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:04:49 +0600
Subject: [PATCH 0475/1515] [escapist] Modernize

---
 youtube_dl/extractor/escapist.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index dadfaa6a5..2cd3af142 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -9,6 +9,7 @@ from ..utils import (
     determine_ext,
     clean_html,
     int_or_none,
+    float_or_none,
 )
 
 
@@ -65,12 +66,12 @@ class EscapistIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        imsVideo = self._parse_json(
+        ims_video = self._parse_json(
             self._search_regex(
                 r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
             video_id)
-        video_id = imsVideo['videoID']
-        key = imsVideo['hash']
+        video_id = ims_video['videoID']
+        key = ims_video['hash']
 
         config_req = compat_urllib_request.Request(
             'http://www.escapistmagazine.com/videos/'
@@ -80,8 +81,11 @@ class EscapistIE(InfoExtractor):
 
         data = json.loads(_decrypt_config(key, config))
 
-        title = clean_html(data['videoData']['title'])
-        duration = data['videoData']['duration'] / 1000
+        video_data = data['videoData']
+
+        title = clean_html(video_data['title'])
+        duration = float_or_none(video_data.get('duration'), 1000)
+        uploader = video_data.get('publisher')
 
         formats = [{
             'url': video['src'],
@@ -97,4 +101,5 @@ class EscapistIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
             'duration': duration,
+            'uploader': uploader,
         }

From b2f82360d78696daca576f558f585b2285db0341 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:06:07 +0600
Subject: [PATCH 0476/1515] [escapist] Add uploader to tests

---
 youtube_dl/extractor/escapist.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 2cd3af142..c85b4c458 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -48,6 +48,7 @@ class EscapistIE(InfoExtractor):
             'title': "Breaking Down Baldur's Gate",
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 264,
+            'uploader': 'The Escapist',
         }
     }, {
         'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
@@ -59,6 +60,7 @@ class EscapistIE(InfoExtractor):
             'title': 'Evolve - One vs Multiplayer',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 304,
+            'uploader': 'The Escapist',
         }
     }]
 

From dc1eed93be7ac2afb3f52237ae0034d24715b4bd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 4 May 2015 15:12:48 +0200
Subject: [PATCH 0477/1515] release 2015.05.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6950afc47..a5a81bcd2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.03'
+__version__ = '2015.05.04'

From 0fe2ff78e68ec03d56bf3d9434eb612ffb683977 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 21:53:05 +0800
Subject: [PATCH 0478/1515] [NBC] Enhance embedURL extraction (closes #2549)

---
 test/test_utils.py          |  5 +++++
 youtube_dl/extractor/nbc.py | 11 +++++++++--
 youtube_dl/utils.py         |  8 ++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 6906a65c2..032d3656a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     unified_strdate,
     unsmuggle_url,
     uppercase_escape,
+    lowercase_escape,
     url_basename,
     urlencode_postdata,
     version_tuple,
@@ -418,6 +419,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
 
+    def test_lowercase_escape(self):
+        self.assertEqual(lowercase_escape('aä'), 'aä')
+        self.assertEqual(lowercase_escape('\\u0026'), '&')
+
     def test_limit_length(self):
         self.assertEqual(limit_length(None, 12), None)
         self.assertEqual(limit_length('foo', 12), 'foo')
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 6cbe03d0f..dc2091be0 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -10,6 +10,8 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     find_xpath_attr,
+    lowercase_escape,
+    unescapeHTML,
 )
 
 
@@ -46,18 +48,23 @@ class NBCIE(InfoExtractor):
                 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
             },
             'skip': 'Only works from US',
+        },
+        {
+            # This video has expired but with an escaped embedURL
+            'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
+            'skip': 'Expired'
         }
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = self._search_regex(
+        theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
             [
                 r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
                 r'"embedURL"\s*:\s*"([^"]+)"'
             ],
-            webpage, 'theplatform url').replace('_no_endcard', '')
+            webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
         return self.url_result(theplatform_url)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a5a5c317e..1013f7c18 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1486,6 +1486,14 @@ def uppercase_escape(s):
         s)
 
 
+def lowercase_escape(s):
+    unicode_escape = codecs.getdecoder('unicode_escape')
+    return re.sub(
+        r'\\u[0-9a-fA-F]{4}',
+        lambda m: unicode_escape(m.group(0))[0],
+        s)
+
+
 def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, compat_str):

From 883340c10758abf76a250172deb12b528b8da7b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 May 2015 16:50:22 +0200
Subject: [PATCH 0479/1515] [livestream:original] Fix extraction (fixes #4702)

---
 youtube_dl/extractor/livestream.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index ec309dadd..5be59a92f 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -202,13 +202,9 @@ class LivestreamOriginalIE(InfoExtractor):
         'url': 'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
         'info_dict': {
             'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
         },
-        'params': {
-            # rtmp
-            'skip_download': True,
-        },
     }, {
         'url': 'https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         'info_dict': {
@@ -221,19 +217,17 @@ class LivestreamOriginalIE(InfoExtractor):
         api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
 
         info = self._download_xml(api_url, video_id)
+        # this url is used on mobile devices
+        stream_url = 'http://x{0}x.api.channel.livestream.com/3.0/getstream.json?id={1}'.format(user, video_id)
+        stream_info = self._download_json(stream_url, video_id)
         item = info.find('channel').find('item')
         ns = {'media': 'http://search.yahoo.com/mrss'}
         thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url']
-        # Remove the extension and number from the path (like 1.jpg)
-        path = self._search_regex(r'(user-files/.+)_.*?\.jpg$', thumbnail_url, 'path')
 
         return {
             'id': video_id,
             'title': item.find('title').text,
-            'url': 'rtmp://extondemand.livestream.com/ondemand',
-            'play_path': 'trans/dv15/mogulus-{0}'.format(path),
-            'player_url': 'http://static.livestream.com/chromelessPlayer/v21/playerapi.swf?hash=5uetk&v=0803&classid=D27CDB6E-AE6D-11cf-96B8-444553540000&jsEnabled=false&wmode=opaque',
-            'ext': 'flv',
+            'url': stream_info['progressiveUrl'],
             'thumbnail': thumbnail_url,
         }
 

From a90552663e555c89638706141c31b0022a9ba27b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 May 2015 16:54:01 +0200
Subject: [PATCH 0480/1515] [livestream:original] Update url format (fixes
 #5598)

---
 youtube_dl/extractor/livestream.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 5be59a92f..6d7733e41 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -194,19 +194,19 @@ class LivestreamIE(InfoExtractor):
 # The original version of Livestream uses a different system
 class LivestreamOriginalIE(InfoExtractor):
     IE_NAME = 'livestream:original'
-    _VALID_URL = r'''(?x)https?://www\.livestream\.com/
+    _VALID_URL = r'''(?x)https?://original\.livestream\.com/
         (?P<user>[^/]+)/(?P<type>video|folder)
         (?:\?.*?Id=|/)(?P<id>.*?)(&|$)
         '''
     _TESTS = [{
-        'url': 'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
+        'url': 'http://original.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
         'info_dict': {
             'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
             'ext': 'mp4',
             'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
         },
     }, {
-        'url': 'https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
+        'url': 'https://original.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         'info_dict': {
             'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         },

From 50aa43b3ae074781bb7c7ad55a85f54b1fd84146 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 22:32:57 +0800
Subject: [PATCH 0481/1515] [nytimes] Implement extracting videos from articles
 (closes  #5436)

---
 youtube_dl/extractor/nytimes.py | 79 +++++++++++++++++++++++----------
 1 file changed, 55 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 03f0a4de6..266d8d400 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -8,30 +8,8 @@ from ..utils import (
 )
 
 
-class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
-        'md5': '18a525a510f942ada2720db5f31644c0',
-        'info_dict': {
-            'id': '100000002847155',
-            'ext': 'mov',
-            'title': 'Verbatim: What Is a Photocopier?',
-            'description': 'md5:93603dada88ddbda9395632fdc5da260',
-            'timestamp': 1398631707,
-            'upload_date': '20140427',
-            'uploader': 'Brett Weiner',
-            'duration': 419,
-        }
-    }, {
-        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
+class NYTimesBaseIE(InfoExtractor):
+    def _extract_video_from_id(self, video_id):
         video_data = self._download_json(
             'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
             video_id, 'Downloading video JSON')
@@ -81,3 +59,56 @@ class NYTimesIE(InfoExtractor):
             'formats': formats,
             'thumbnails': thumbnails,
         }
+
+
+class NYTimesIE(NYTimesBaseIE):
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
+        'md5': '18a525a510f942ada2720db5f31644c0',
+        'info_dict': {
+            'id': '100000002847155',
+            'ext': 'mov',
+            'title': 'Verbatim: What Is a Photocopier?',
+            'description': 'md5:93603dada88ddbda9395632fdc5da260',
+            'timestamp': 1398631707,
+            'upload_date': '20140427',
+            'uploader': 'Brett Weiner',
+            'duration': 419,
+        }
+    }, {
+        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        return self._extract_video_from_id(video_id)
+
+
+class NYTimesArticleIE(NYTimesBaseIE):
+    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/\d{4}/\d{2}/\d{2}/(?:[^/]+/)*(?P<id>[^.]+)\.html'
+    _TEST = {
+        'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
+        'md5': 'e2076d58b4da18e6a001d53fd56db3c9',
+        'info_dict': {
+            'id': '100000003628438',
+            'ext': 'mov',
+            'title': 'New Minimum Wage: $70,000 a Year',
+            'description': 'Dan Price, C.E.O. of Gravity Payments, surprised his 120-person staff by announcing that he planned over the next three years to raise the salary of every employee to $70,000 a year.',
+            'timestamp': 1429033037,
+            'upload_date': '20150414',
+            'uploader': 'Matthew Williams',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._html_search_regex(r'data-videoid="(\d+)"', webpage, 'video id')
+
+        return self._extract_video_from_id(video_id)

From df8418ffcff60c5a90e3138a39959d36cebcee7f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 23:00:09 +0800
Subject: [PATCH 0482/1515] [nytimes] Extend _VALID_URL (#2754)

---
 youtube_dl/extractor/__init__.py | 5 ++++-
 youtube_dl/extractor/nytimes.py  | 9 ++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 41af925cc..1c3a46dd8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -363,7 +363,10 @@ from .nrk import (
 )
 from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
-from .nytimes import NYTimesIE
+from .nytimes import (
+    NYTimesIE,
+    NYTimesArticleIE,
+)
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 266d8d400..6ffbe3863 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -89,8 +89,8 @@ class NYTimesIE(NYTimesBaseIE):
 
 
 class NYTimesArticleIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/\d{4}/\d{2}/\d{2}/(?:[^/]+/)*(?P<id>[^.]+)\.html'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
+    _TESTS = [{
         'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
         'md5': 'e2076d58b4da18e6a001d53fd56db3c9',
         'info_dict': {
@@ -102,7 +102,10 @@ class NYTimesArticleIE(NYTimesBaseIE):
             'upload_date': '20150414',
             'uploader': 'Matthew Williams',
         }
-    }
+    }, {
+        'url': 'http://www.nytimes.com/news/minute/2014/03/17/times-minute-whats-next-in-crimea/?_php=true&_type=blogs&_php=true&_type=blogs&_r=1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From dfad3aac981a311de89a5ae29121bcb6255902f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 21:23:26 +0600
Subject: [PATCH 0483/1515] [rutv] Fix live stream test URL

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 1ec2c86e5..6b56b1d49 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -87,7 +87,7 @@ class RUTVIE(InfoExtractor):
             'skip': 'Translation has finished',
         },
         {
-            'url': 'http://live.russia.tv/index/index/channel_id/3',
+            'url': 'http://player.rutv.ru/iframe/live/id/21/showZoomBtn/false/isPlay/true/',
             'info_dict': {
                 'id': '21',
                 'ext': 'mp4',

From e038d5c4e323bdfa9de02ad59d1caa2c819d9cf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 21:29:32 +0600
Subject: [PATCH 0484/1515] [rutv] Fix preference

---
 youtube_dl/extractor/rutv.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 6b56b1d49..a5e98415f 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -128,8 +128,10 @@ class RUTVIE(InfoExtractor):
         elif video_path.startswith('index/iframe/cast_id'):
             video_type = 'live'
 
+        is_live = video_type == 'live'
+
         json_data = self._download_json(
-            'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if video_type == 'live' else '', video_id),
+            'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if is_live else '', video_id),
             video_id, 'Downloading JSON')
 
         if json_data['errors']:
@@ -156,6 +158,7 @@ class RUTVIE(InfoExtractor):
 
         for transport, links in media['sources'].items():
             for quality, url in links.items():
+                preference = -1 if priority_transport == transport else -2
                 if transport == 'rtmp':
                     mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>.+)$', url)
                     if not mobj:
@@ -169,9 +172,11 @@ class RUTVIE(InfoExtractor):
                         'rtmp_live': True,
                         'ext': 'flv',
                         'vbr': int(quality),
+                        'preference': preference,
                     }
                 elif transport == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(url, video_id, 'mp4'))
+                    formats.extend(self._extract_m3u8_formats(
+                        url, video_id, 'mp4', preference=preference, m3u8_id='hls'))
                     continue
                 else:
                     fmt = {
@@ -181,7 +186,6 @@ class RUTVIE(InfoExtractor):
                     'width': width,
                     'height': height,
                     'format_id': '%s-%s' % (transport, quality),
-                    'preference': -1 if priority_transport == transport else -2,
                 })
                 formats.append(fmt)
 
@@ -190,8 +194,6 @@ class RUTVIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        is_live = video_type == 'live'
-
         return {
             'id': video_id,
             'title': self._live_title(title) if is_live else title,

From 1aa43d77c084bc2dd735fcd3d03b8b2fc3407c51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 21:29:56 +0600
Subject: [PATCH 0485/1515] [rutv] Remove superfluous check

---
 youtube_dl/extractor/rutv.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a5e98415f..55604637d 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -189,9 +189,6 @@ class RUTVIE(InfoExtractor):
                 })
                 formats.append(fmt)
 
-        if not formats:
-            raise ExtractorError('No media links available for %s' % video_id)
-
         self._sort_formats(formats)
 
         return {

From 7212560f4d6b0de5b76eb41090c639855915946e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= <aurelien@tassatux.net>
Date: Wed, 29 Apr 2015 01:07:33 +0200
Subject: [PATCH 0486/1515] [noco] Retrieve video language according to user
 options

---
 youtube_dl/extractor/noco.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 251e6da07..20a658149 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -86,22 +86,36 @@ class NocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        options = self._call_api('users/init', None, 'Downloading user options JSON')['options']
+        audio_lang = options.get('audio_language', 'fr')
+
         medias = self._call_api(
             'shows/%s/medias' % video_id,
             video_id, 'Downloading video JSON')
 
+        show = self._call_api(
+            'shows/by_id/%s' % video_id,
+            video_id, 'Downloading show JSON')[0]
+
+        if audio_lang == 'original':
+            audio_lang = show['original_lang']
+        if len(medias) == 1:
+            audio_lang = list(medias.keys())[0]
+        elif not audio_lang in medias:
+            audio_lang = 'fr'
+
         qualities = self._call_api(
             'qualities',
             video_id, 'Downloading qualities JSON')
 
         formats = []
 
-        for lang, lang_dict in medias['fr']['video_list'].items():
+        for lang, lang_dict in medias[audio_lang]['video_list'].items():
             for format_id, fmt in lang_dict['quality_list'].items():
                 format_id_extended = '%s-%s' % (lang, format_id) if lang != 'none' else format_id
 
                 video = self._call_api(
-                    'shows/%s/video/%s/fr' % (video_id, format_id.lower()),
+                    'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
                     video_id, 'Downloading %s video JSON' % format_id_extended,
                     lang if lang != 'none' else None)
 
@@ -127,10 +141,6 @@ class NocoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        show = self._call_api(
-            'shows/by_id/%s' % video_id,
-            video_id, 'Downloading show JSON')[0]
-
         upload_date = unified_strdate(show['online_date_start_utc'])
         uploader = show['partner_name']
         uploader_id = show['partner_key']

From ff9d68e7be2dd01a21cc0ed90aaa594b5d36697f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= <aurelien@tassatux.net>
Date: Mon, 4 May 2015 19:55:29 +0200
Subject: [PATCH 0487/1515] [noco] Add test for multi languages video

---
 youtube_dl/extractor/noco.py | 45 +++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 20a658149..f86d210ee 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -25,21 +25,38 @@ class NocoIE(InfoExtractor):
     _SUB_LANG_TEMPLATE = '&sub_lang=%s'
     _NETRC_MACHINE = 'noco'
 
-    _TEST = {
-        'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
-        'md5': '0a993f0058ddbcd902630b2047ef710e',
-        'info_dict': {
-            'id': '11538',
-            'ext': 'mp4',
-            'title': 'Ami Ami Idol - Hello! France',
-            'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
-            'upload_date': '20140412',
-            'uploader': 'Nolife',
-            'uploader_id': 'NOL',
-            'duration': 2851.2,
+    _TESTS = [
+        {
+            'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
+            'md5': '0a993f0058ddbcd902630b2047ef710e',
+            'info_dict': {
+                'id': '11538',
+                'ext': 'mp4',
+                'title': 'Ami Ami Idol - Hello! France',
+                'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
+                'upload_date': '20140412',
+                'uploader': 'Nolife',
+                'uploader_id': 'NOL',
+                'duration': 2851.2,
+            },
+            'skip': 'Requires noco account',
         },
-        'skip': 'Requires noco account',
-    }
+        {
+            'url': 'http://noco.tv/emission/12610/lbl42/the-guild/s01e01-wake-up-call',
+            'md5': 'c190f1f48e313c55838f1f412225934d',
+            'info_dict': {
+                'id': '12610',
+                'ext': 'mp4',
+                'title': 'The Guild #1 - Wake-Up Call',
+                'description': '',
+                'upload_date': '20140627',
+                'uploader': 'LBL42',
+                'uploader_id': 'LBL',
+                'duration': 233.023,
+            },
+            'skip': 'Requires noco account',
+        }
+    ]
 
     def _real_initialize(self):
         self._login()

From 6568382d6f3a986a773aa4d92b6bcbd367ccb794 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:27:24 +0600
Subject: [PATCH 0488/1515] [noco] Extract all variations of audio/subtitles
 media

---
 youtube_dl/extractor/noco.py | 69 +++++++++++++++++++-----------------
 1 file changed, 37 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index f86d210ee..e44ece5a2 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -103,9 +103,6 @@ class NocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        options = self._call_api('users/init', None, 'Downloading user options JSON')['options']
-        audio_lang = options.get('audio_language', 'fr')
-
         medias = self._call_api(
             'shows/%s/medias' % video_id,
             video_id, 'Downloading video JSON')
@@ -114,12 +111,17 @@ class NocoIE(InfoExtractor):
             'shows/by_id/%s' % video_id,
             video_id, 'Downloading show JSON')[0]
 
-        if audio_lang == 'original':
-            audio_lang = show['original_lang']
+        options = self._call_api(
+            'users/init', video_id,
+            'Downloading user options JSON')['options']
+        audio_lang_pref = options.get('audio_language') or options.get('language', 'fr')
+
+        if audio_lang_pref == 'original':
+            audio_lang_pref = show['original_lang']
         if len(medias) == 1:
-            audio_lang = list(medias.keys())[0]
-        elif not audio_lang in medias:
-            audio_lang = 'fr'
+            audio_lang_pref = list(medias.keys())[0]
+        elif audio_lang_pref not in medias:
+            audio_lang_pref = 'fr'
 
         qualities = self._call_api(
             'qualities',
@@ -127,34 +129,37 @@ class NocoIE(InfoExtractor):
 
         formats = []
 
-        for lang, lang_dict in medias[audio_lang]['video_list'].items():
-            for format_id, fmt in lang_dict['quality_list'].items():
-                format_id_extended = '%s-%s' % (lang, format_id) if lang != 'none' else format_id
+        for audio_lang, audio_lang_dict in medias.items():
+            preference = 1 if audio_lang == audio_lang_pref else 0
+            for sub_lang, lang_dict in audio_lang_dict['video_list'].items():
+                for format_id, fmt in lang_dict['quality_list'].items():
+                    format_id_extended = 'audio-%s_sub-%s_%s' % (audio_lang, sub_lang, format_id)
 
-                video = self._call_api(
-                    'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
-                    video_id, 'Downloading %s video JSON' % format_id_extended,
-                    lang if lang != 'none' else None)
+                    video = self._call_api(
+                        'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
+                        video_id, 'Downloading %s video JSON' % format_id_extended,
+                        sub_lang if sub_lang != 'none' else None)
 
-                file_url = video['file']
-                if not file_url:
-                    continue
+                    file_url = video['file']
+                    if not file_url:
+                        continue
 
-                if file_url in ['forbidden', 'not found']:
-                    popmessage = video['popmessage']
-                    self._raise_error(popmessage['title'], popmessage['message'])
+                    if file_url in ['forbidden', 'not found']:
+                        popmessage = video['popmessage']
+                        self._raise_error(popmessage['title'], popmessage['message'])
 
-                formats.append({
-                    'url': file_url,
-                    'format_id': format_id_extended,
-                    'width': fmt['res_width'],
-                    'height': fmt['res_lines'],
-                    'abr': fmt['audiobitrate'],
-                    'vbr': fmt['videobitrate'],
-                    'filesize': fmt['filesize'],
-                    'format_note': qualities[format_id]['quality_name'],
-                    'preference': qualities[format_id]['priority'],
-                })
+                    formats.append({
+                        'url': file_url,
+                        'format_id': format_id_extended,
+                        'width': fmt['res_width'],
+                        'height': fmt['res_lines'],
+                        'abr': fmt['audiobitrate'],
+                        'vbr': fmt['videobitrate'],
+                        'filesize': fmt['filesize'],
+                        'format_note': qualities[format_id]['quality_name'],
+                        'quality': qualities[format_id]['priority'],
+                        'preference': preference,
+                    })
 
         self._sort_formats(formats)
 

From 815ac0293ee9d7f3771499cd71d833c83575bec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:38:13 +0600
Subject: [PATCH 0489/1515] [noco] Modernize

---
 youtube_dl/extractor/noco.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index e44ece5a2..098c564bd 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -14,6 +14,9 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
     unified_strdate,
 )
 
@@ -151,22 +154,22 @@ class NocoIE(InfoExtractor):
                     formats.append({
                         'url': file_url,
                         'format_id': format_id_extended,
-                        'width': fmt['res_width'],
-                        'height': fmt['res_lines'],
-                        'abr': fmt['audiobitrate'],
-                        'vbr': fmt['videobitrate'],
-                        'filesize': fmt['filesize'],
-                        'format_note': qualities[format_id]['quality_name'],
-                        'quality': qualities[format_id]['priority'],
+                        'width': int_or_none(fmt.get('res_width')),
+                        'height': int_or_none(fmt.get('res_lines')),
+                        'abr': int_or_none(fmt.get('audiobitrate')),
+                        'vbr': int_or_none(fmt.get('videobitrate')),
+                        'filesize': int_or_none(fmt.get('filesize')),
+                        'format_note': qualities[format_id].get('quality_name'),
+                        'quality': qualities[format_id].get('priority'),
                         'preference': preference,
                     })
 
         self._sort_formats(formats)
 
-        upload_date = unified_strdate(show['online_date_start_utc'])
-        uploader = show['partner_name']
-        uploader_id = show['partner_key']
-        duration = show['duration_ms'] / 1000.0
+        timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')
+        uploader = show.get('partner_name')
+        uploader_id = show.get('partner_key')
+        duration = float_or_none(show.get('duration_ms'), 1000)
 
         thumbnails = []
         for thumbnail_key, thumbnail_url in show.items():
@@ -198,7 +201,7 @@ class NocoIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnails': thumbnails,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'duration': duration,

From 01e4b1ee14a3e9dedcb6a156c6eaf1603a8a0592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:50:39 +0600
Subject: [PATCH 0490/1515] [noco] Update tests

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 098c564bd..5674ee2a4 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -51,7 +51,7 @@ class NocoIE(InfoExtractor):
                 'id': '12610',
                 'ext': 'mp4',
                 'title': 'The Guild #1 - Wake-Up Call',
-                'description': '',
+                'timestamp': 1403863200,
                 'upload_date': '20140627',
                 'uploader': 'LBL42',
                 'uploader_id': 'LBL',

From ce5c1ae51703d1be4e486cbe8d3dfc2ab16372a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:52:21 +0600
Subject: [PATCH 0491/1515] [noco] Remove unused import

---
 youtube_dl/extractor/noco.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 5674ee2a4..664dc81d4 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -17,7 +17,6 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_iso8601,
-    unified_strdate,
 )
 
 

From e10dc0e1f0f6b33b60a601bf7855e76e7813ecde Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Tue, 5 May 2015 08:59:09 +0200
Subject: [PATCH 0492/1515] [southparknl] Add extractor for southpark.nl

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 15 +++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1c3a46dd8..b042e9215 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -485,6 +485,7 @@ from .southpark import (
     SouthParkIE,
     SouthParkEsIE,
     SouthparkDeIE,
+    SouthParkNlIE
 )
 from .space import SpaceIE
 from .spankbang import SpankBangIE
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index e3b73295c..ebcd2ed04 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -46,3 +46,18 @@ class SouthparkDeIE(SouthParkIE):
             'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
         },
     }]
+
+
+class SouthParkNlIE(SouthParkIE):
+    IE_NAME = 'southpark.nl'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.nl/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _FEED_URL = 'http://www.southpark.nl/feeds/video-player/mrss/'
+
+    _TESTS = [{
+        'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
+        'info_dict': {
+            'id': 'c56b9a1f-0e42-4942-89e0-8e721d814c5c',
+            'ext': 'mp4',
+            'title': 'South Park 1806: Freemium Isn\'t Free - Act 3',
+        },
+    }]

From 3408f6e64a7438dbc1bcc087a00e8540ef660eb3 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Tue, 5 May 2015 09:01:07 +0200
Subject: [PATCH 0493/1515] [southparkde] Fix naming inconsistency

The class was first called 'SouthparkDe'. It is now changed to
'SouthParkDe' to match the name of the other extractors.
---
 youtube_dl/extractor/__init__.py  | 2 +-
 youtube_dl/extractor/southpark.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b042e9215..5dfa781f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -484,7 +484,7 @@ from .soundgasm import (
 from .southpark import (
     SouthParkIE,
     SouthParkEsIE,
-    SouthparkDeIE,
+    SouthParkDeIE,
     SouthParkNlIE
 )
 from .space import SpaceIE
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index ebcd2ed04..2b12f2ad3 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -32,7 +32,7 @@ class SouthParkEsIE(SouthParkIE):
     }]
 
 
-class SouthparkDeIE(SouthParkIE):
+class SouthParkDeIE(SouthParkIE):
     IE_NAME = 'southpark.de'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.de/(?:clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'

From dcf807790673ef8adb22ed6ff93f6232903d75f6 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Tue, 5 May 2015 09:17:21 +0200
Subject: [PATCH 0494/1515] [southparknl] Fix test to match playlist tests

---
 youtube_dl/extractor/southpark.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 2b12f2ad3..59e31198c 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -55,9 +55,5 @@ class SouthParkNlIE(SouthParkIE):
 
     _TESTS = [{
         'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
-        'info_dict': {
-            'id': 'c56b9a1f-0e42-4942-89e0-8e721d814c5c',
-            'ext': 'mp4',
-            'title': 'South Park 1806: Freemium Isn\'t Free - Act 3',
-        },
+        'playlist_count': 4,
     }]

From 07d2921c6d13ce43adb9347677e50c36268fe54f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 5 May 2015 23:39:54 +0800
Subject: [PATCH 0495/1515] [lifenews] Correctly determine iframe links (fixes
 #5618)

---
 youtube_dl/extractor/lifenews.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 330138692..963f16e1a 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -39,6 +39,17 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150402',
             'uploader': 'embed.life.ru',
         }
+    }, {
+        'url': 'http://lifenews.ru/news/153461',
+        'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
+        'info_dict': {
+            'id': '153461',
+            'ext': 'mp4',
+            'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
+            'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+            'upload_date': '20150505',
+            'uploader': 'embed.life.ru',
+        }
     }]
 
     def _real_extract(self, url):
@@ -88,6 +99,8 @@ class LifeNewsIE(InfoExtractor):
             return cur_info
 
         if iframe_link:
+            if iframe_link.startswith('//'):
+                iframe_link = 'http:' + iframe_link
             cur_info = dict(common_info)
             cur_info.update({
                 '_type': 'url_transparent',

From b326b07adc49466a4bdd0ea3c63f329a9e523121 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 21:49:36 +0600
Subject: [PATCH 0496/1515] [lifenews] Use `_proto_relative_url`

---
 youtube_dl/extractor/lifenews.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 963f16e1a..081016b80 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -99,8 +99,7 @@ class LifeNewsIE(InfoExtractor):
             return cur_info
 
         if iframe_link:
-            if iframe_link.startswith('//'):
-                iframe_link = 'http:' + iframe_link
+            iframe_link = self._proto_relative_url(iframe_link, 'http:')
             cur_info = dict(common_info)
             cur_info.update({
                 '_type': 'url_transparent',

From d6a1738892038940ef4af59a33aeddc99ef0c966 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Wed, 6 May 2015 11:48:36 +0100
Subject: [PATCH 0497/1515] [archive.org] Fix incorrect url condition (closes
 #5628)

The condition for assigning to json_url is the wrong way round:

currently for url: aaa.com/xxx

we get:

aaa.com/xxx&output=json

instead of the correct value:

aaa.com/xxx?output=json
---
 youtube_dl/extractor/archiveorg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 9fc35a42b..8feb7cb74 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -33,7 +33,7 @@ class ArchiveOrgIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = url + ('?' if '?' in url else '&') + 'output=json'
+        json_url = url + ('&' if '?' in url else '?') + 'output=json'
         data = self._download_json(json_url, video_id)
 
         def get_optional(data_dict, field):

From 74f728249fcf58b0c99eb69606b04b304d882423 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:24:24 +0600
Subject: [PATCH 0498/1515] [extractor/common] Fallback to empty string for
 (yet) missing `format_id` in `_sort_formats` (Closes #5624)

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3ae5d5212..34e27a35a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -764,7 +764,7 @@ class InfoExtractor(object):
                 f.get('fps') if f.get('fps') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,
-                f.get('format_id'),
+                f.get('format_id') if f.get('format_id') is not None else '',
             )
         formats.sort(key=_formats_key)
 

From e8b9ab8957084d1b7ab60f14574e5d14d9417909 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:31:25 +0600
Subject: [PATCH 0499/1515] [pbs] Add `format_id` for direct links

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 761bd6d8d..143a76696 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -187,6 +187,7 @@ class PBSIE(InfoExtractor):
             else:
                 formats.append({
                     'url': format_url,
+                    'format_id': redirect.get('eeid'),
                 })
         self._sort_formats(formats)
 

From 6a8f9cd22efa7db844e035a3e40571f7d05eebea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:39:53 +0600
Subject: [PATCH 0500/1515] [giga] Fix view count extraction

---
 youtube_dl/extractor/giga.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/giga.py b/youtube_dl/extractor/giga.py
index 775890112..28eb733e2 100644
--- a/youtube_dl/extractor/giga.py
+++ b/youtube_dl/extractor/giga.py
@@ -85,7 +85,8 @@ class GigaIE(InfoExtractor):
             r'class="author">([^<]+)</a>', webpage, 'uploader', fatal=False)
 
         view_count = str_to_int(self._search_regex(
-            r'<span class="views"><strong>([\d.]+)</strong>', webpage, 'view count', fatal=False))
+            r'<span class="views"><strong>([\d.,]+)</strong>',
+            webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,

From 1ed34f3dd6f534a3a5a3d6808d8d3466a9e5dea2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:43:36 +0600
Subject: [PATCH 0501/1515] [gorillavid] Switch 404 test to only matching

---
 youtube_dl/extractor/gorillavid.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index ed2623456..6147596e4 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -35,13 +35,7 @@ class GorillaVidIE(InfoExtractor):
         },
     }, {
         'url': 'http://gorillavid.in/embed-z08zf8le23c6-960x480.html',
-        'md5': 'c9e293ca74d46cad638e199c3f3fe604',
-        'info_dict': {
-            'id': 'z08zf8le23c6',
-            'ext': 'mp4',
-            'title': 'Say something nice',
-            'thumbnail': 're:http://.*\.jpg',
-        },
+        'only_matching': True,
     }, {
         'url': 'http://daclips.in/3rso4kdn6f9m',
         'md5': '1ad8fd39bb976eeb66004d3a4895f106',

From ad0c0ad3b4db546dc2078391f397aa5b44c8150b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:52:26 +0600
Subject: [PATCH 0502/1515] [historicfilms] Fix tape id extraction

---
 youtube_dl/extractor/historicfilms.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/historicfilms.py b/youtube_dl/extractor/historicfilms.py
index 40afbe537..6a36933ac 100644
--- a/youtube_dl/extractor/historicfilms.py
+++ b/youtube_dl/extractor/historicfilms.py
@@ -25,7 +25,8 @@ class HistoricFilmsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         tape_id = self._search_regex(
-            r'class="tapeId">([^<]+)<', webpage, 'tape id')
+            [r'class="tapeId"[^>]*>([^<]+)<', r'tapeId\s*:\s*"([^"]+)"'],
+            webpage, 'tape id')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)

From ac6c358c2ab882427f74af47fe5df762dd348c20 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 02:08:47 +0800
Subject: [PATCH 0503/1515] [teamcoco] Fix extracting preload data again

---
 youtube_dl/extractor/teamcoco.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 2381676b4..95d58ddd0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import binascii
 import re
 
 from .common import InfoExtractor
@@ -9,6 +10,7 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
+from ..compat import compat_ord
 
 
 class TeamcocoIE(InfoExtractor):
@@ -66,7 +68,7 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        preload = None
+        data = preload = None
         preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
         if preloads:
             preload = max([(len(p), p) for p in preloads])[1]
@@ -80,11 +82,27 @@ class TeamcocoIE(InfoExtractor):
             ], webpage.replace('","', ''), 'preload data', default=None)
 
         if not preload:
+            preload_codes = self._html_search_regex(
+                r'(function.+)setTimeout\(function\(\)\{playlist',
+                webpage, 'preload codes')
+            base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+            base64_fragments.remove('init')
+            for i in range(len(base64_fragments)):
+                cur_sequence = (''.join(base64_fragments[i:] + base64_fragments[:i])).encode('ascii')
+                try:
+                    raw_data = base64.b64decode(cur_sequence)
+                except (TypeError, binascii.Error):
+                    continue
+                if compat_ord(raw_data[0]) == compat_ord('{'):
+                    data = self._parse_json(raw_data.decode('utf-8'), video_id, fatal=False)
+
+        if not preload and not data:
             raise ExtractorError(
                 'Preload information could not be extracted', expected=True)
 
-        data = self._parse_json(
-            base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
+        if not data:
+            data = self._parse_json(
+                base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From d9a743d9178b0ed1e44168e42e8cec2d7dd8d63e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:05:37 +0800
Subject: [PATCH 0504/1515] [vice] Remove a redundant print

---
 youtube_dl/extractor/vice.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 71f520fb5..04e2b0ba7 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -31,7 +31,6 @@ class ViceIE(InfoExtractor):
                 r'embedCode=([^&\'"]+)', webpage,
                 'ooyala embed code')
             ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
-            print(ooyala_url)
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
         return self.url_result(ooyala_url, ie='Ooyala')

From 05d5392cdaa558dba285c328182d4f3e82fb8e8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:06:22 +0800
Subject: [PATCH 0505/1515] [common] Ignore subtitles in m3u8

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 34e27a35a..981e34bc7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -896,7 +896,7 @@ class InfoExtractor(object):
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                last_media_name = last_media.get('NAME') if last_media else None
+                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
                 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),

From 84bf31aaf8b9b7397de5f3189295d93e8e93e5e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:12:01 +0800
Subject: [PATCH 0506/1515] [ooyala] Extract m3u8 information (#2292)

---
 youtube_dl/extractor/ooyala.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index d5b05c18f..b33e8230d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     unescapeHTML,
     ExtractorError,
+    determine_ext,
 )
 
 
@@ -44,11 +45,21 @@ class OoyalaIE(InfoExtractor):
                               ie=cls.ie_key())
 
     def _extract_result(self, info, more_info):
+        embedCode = info['embedCode']
+        video_url = info.get('ipad_url') or info['url']
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': 'mp4',
+            }]
+
         return {
-            'id': info['embedCode'],
-            'ext': 'mp4',
+            'id': embedCode,
             'title': unescapeHTML(info['title']),
-            'url': info.get('ipad_url') or info['url'],
+            'formats': formats,
             'description': unescapeHTML(more_info['description']),
             'thumbnail': more_info['promo'],
         }

From c09593c04e0b345df02cae663dc064d29e241cba Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 15:07:11 +0100
Subject: [PATCH 0507/1515] [BildIE] Escape ampersands in xml and update test
 thumbnail

---
 youtube_dl/extractor/bild.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 77b562d99..ba0c185eb 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+  int_or_none,
+  fix_xml_ampersands,
+)
 
 
 class BildIE(InfoExtractor):
@@ -15,7 +18,7 @@ class BildIE(InfoExtractor):
             'id': '38184146',
             'ext': 'mp4',
             'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
+            'thumbnail': 'http://bilder.bild.de/fotos/bild-hat-sie-getestet-das-koennen-apples-neue-ipads-38184138/Bild/1.bild.jpg',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }
@@ -25,7 +28,7 @@ class BildIE(InfoExtractor):
         video_id = self._match_id(url)
 
         xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
-        doc = self._download_xml(xml_url, video_id)
+        doc = self._download_xml(xml_url, video_id, transform_source=fix_xml_ampersands)
 
         duration = int_or_none(doc.attrib.get('duration'), scale=1000)
 

From aafe2739909882931f7f624e83fe532af0bfafc1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 22:07:32 +0800
Subject: [PATCH 0508/1515] [ooyala] Use SAS API to extract info (fixes #4336)

---
 youtube_dl/extractor/ooyala.py | 43 ++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index b33e8230d..0b049274a 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -1,12 +1,14 @@
 from __future__ import unicode_literals
 import re
 import json
+import base64
 
 from .common import InfoExtractor
 from ..utils import (
     unescapeHTML,
     ExtractorError,
     determine_ext,
+    int_or_none,
 )
 
 
@@ -33,6 +35,17 @@ class OoyalaIE(InfoExtractor):
                 'description': '',
             },
         },
+        {
+            # Information available only through SAS api
+            # From http://community.plm.automation.siemens.com/t5/News-NX-Manufacturing/Tool-Path-Divide/ba-p/4187
+            'url': 'http://player.ooyala.com/player.js?embedCode=FiOG81ZTrvckcchQxmalf4aQj590qTEx',
+            'md5': 'a84001441b35ea492bc03736e59e7935',
+            'info_dict': {
+                'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
+                'ext': 'mp4',
+                'title': 'Ooyala video',
+            }
+        }
     ]
 
     @staticmethod
@@ -88,6 +101,36 @@ class OoyalaIE(InfoExtractor):
                 mobile_player, 'info', fatal=False, default=None)
             if videos_info:
                 break
+
+        if not videos_info:
+            formats = []
+            auth_data = self._download_json(
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                embedCode)
+
+            cur_auth_data = auth_data['authorization_data'][embedCode]
+
+            for stream in cur_auth_data['streams']:
+                formats.append({
+                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
+                    'ext': stream.get('delivery_type'),
+                    'format': stream.get('video_codec'),
+                    'format_id': stream.get('profile'),
+                    'width': int_or_none(stream.get('width')),
+                    'height': int_or_none(stream.get('height')),
+                    'abr': int_or_none(stream.get('audio_bitrate')),
+                    'vbr': int_or_none(stream.get('video_bitrate')),
+                })
+            if len(formats):
+                return {
+                    'id': embedCode,
+                    'formats': formats,
+                    'title': 'Ooyala video',
+                }
+
+            if not cur_auth_data['authorized']:
+                raise ExtractorError(cur_auth_data['message'], expected=True)
+
         if not videos_info:
             raise ExtractorError('Unable to extract info')
         videos_info = videos_info.replace('\\"', '"')

From bc08873cff6d36ba175e5121b0ba1ad270c664c9 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 15:09:27 +0100
Subject: [PATCH 0509/1515] Fix indents

---
 youtube_dl/extractor/bild.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index ba0c185eb..41dd1cbc1 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-  int_or_none,
-  fix_xml_ampersands,
+    int_or_none,
+    fix_xml_ampersands,
 )
 
 

From f22834a3723dd59f1e04c27d0bc3a373ebc17183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 20:20:43 +0600
Subject: [PATCH 0510/1515] [bild] Relax thumbnail test check

---
 youtube_dl/extractor/bild.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 41dd1cbc1..4d8cce1ef 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -18,7 +18,7 @@ class BildIE(InfoExtractor):
             'id': '38184146',
             'ext': 'mp4',
             'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/bild-hat-sie-getestet-das-koennen-apples-neue-ipads-38184138/Bild/1.bild.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }

From 3799834dcff27ffeea66e3ec96166f8da8fa73ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 20:46:11 +0600
Subject: [PATCH 0511/1515] [YoutubeDL] Do not force bestvideo+bestaudio when
 outtmpl is stdout (#5627)

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb7470f72..d8583a8eb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,7 +1086,9 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if info_dict['extractor'] in ['youtube', 'ted'] and FFmpegMergerPP(self).available:
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
+                    and info_dict['extractor'] in ['youtube', 'ted']
+                    and FFmpegMergerPP(self).available):
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 406224be5231e602b543579706ad6056b75fbe68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 21:02:59 +0600
Subject: [PATCH 0512/1515] [extractor/generic] Fix following incomplete
 redirects (#5640)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d09e85665..cd7c47d6d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1453,7 +1453,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = found.group(1)
+                new_url = compat_urlparse.urljoin(url, found.group(1))
                 self.report_following_redirect(new_url)
                 return {
                     '_type': 'url',

From 5268a05e4722d74e125a97b023d92943745bb249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 May 2015 17:04:15 +0200
Subject: [PATCH 0513/1515] [ooyala] Style fix

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 0b049274a..c0e6d643d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -121,7 +121,7 @@ class OoyalaIE(InfoExtractor):
                     'abr': int_or_none(stream.get('audio_bitrate')),
                     'vbr': int_or_none(stream.get('video_bitrate')),
                 })
-            if len(formats):
+            if formats:
                 return {
                     'id': embedCode,
                     'formats': formats,

From 09b412dafa4bc95b9850d77b3bce5f7eac47a578 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 02:12:28 +0800
Subject: [PATCH 0514/1515] [nhl] Partial support for hlg id (fixes #4285)

---
 youtube_dl/extractor/nhl.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 407465998..b572370c2 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -21,6 +21,9 @@ class NHLBaseInfoExtractor(InfoExtractor):
         return json_string.replace('\\\'', '\'')
 
     def _real_extract_video(self, video_id):
+        vid_parts = video_id.split(',')
+        if len(vid_parts) == 3:
+            video_id = '%s0%s%s-X-h' % (vid_parts[0][:4], vid_parts[1], vid_parts[2].rjust(4, '0'))
         json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
         data = self._download_json(
             json_url, video_id, transform_source=self._fix_json)
@@ -60,7 +63,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P<id>[-0-9a-zA-Z,]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -101,6 +104,17 @@ class NHLIE(NHLBaseInfoExtractor):
     }, {
         'url': 'http://video.nhl.com/videocenter/?id=736722',
         'only_matching': True,
+    }, {
+        'url': 'http://video.nhl.com/videocenter/console?hlg=20142015,2,299&lang=en',
+        'md5': '076fcb88c255154aacbf0a7accc3f340',
+        'info_dict': {
+            'id': '2014020299-X-h',
+            'ext': 'mp4',
+            'title': 'Penguins at Islanders / Game Highlights',
+            'description': 'Home broadcast - Pittsburgh Penguins at New York Islanders - November 22, 2014',
+            'duration': 268,
+            'upload_date': '20141122',
+        }
     }]
 
     def _real_extract(self, url):

From 46be82b811d91be0b0876cf141e6a94e65b8fd7f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 7 May 2015 21:58:03 +0300
Subject: [PATCH 0515/1515] [vessel] Use `main_video_asset` when searching for
 video_asset (Fixes #5623)

---
 youtube_dl/extractor/vessel.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 6215f0642..3c8d2a943 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -38,9 +38,13 @@ class VesselIE(InfoExtractor):
         return req
 
     @staticmethod
-    def find_assets(data, asset_type):
+    def find_assets(data, asset_type, asset_id=None):
         for asset in data.get('assets', []):
-            if asset.get('type') == asset_type:
+            if not asset.get('type') == asset_type:
+                continue
+            elif asset_id is not None and not asset.get('id') == asset_id:
+                continue
+            else:
                 yield asset
 
     def _check_access_rights(self, data):
@@ -82,11 +86,13 @@ class VesselIE(InfoExtractor):
         req = VesselIE.make_json_request(
             self._API_URL_TEMPLATE % asset_id, {'client': 'web'})
         data = self._download_json(req, video_id)
+        video_asset_id = data.get('main_video_asset')
 
         self._check_access_rights(data)
 
         try:
-            video_asset = next(VesselIE.find_assets(data, 'video'))
+            video_asset = next(
+                VesselIE.find_assets(data, 'video', asset_id=video_asset_id))
         except StopIteration:
             raise ExtractorError('No video assets found')
 

From 156fc83a55b14258bb4a2fa1ec3b02d4db679603 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 03:02:34 +0800
Subject: [PATCH 0516/1515] [downloader/rtmp] Fix a typo

---
 youtube_dl/downloader/rtmp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6865b5e2f..7d19bb808 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -131,7 +131,7 @@ class RtmpFD(FileDownloader):
         if play_path is not None:
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
-            basic_args += ['--tcUrl', url]
+            basic_args += ['--tcUrl', tc_url]
         if test:
             basic_args += ['--stop', '1']
         if flash_version is not None:

From 7ef00afe9da87c7d0fdbea93af39b47d5447f1a0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 03:09:19 +0800
Subject: [PATCH 0517/1515] [nhl] Support RTMP videos (fixes #4481)

---
 youtube_dl/extractor/nhl.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index b572370c2..279b18386 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -50,7 +50,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
             video_url = initial_video_url
 
         join = compat_urlparse.urljoin
-        return {
+        ret = {
             'id': video_id,
             'title': info['name'],
             'url': video_url,
@@ -59,6 +59,15 @@ class NHLBaseInfoExtractor(InfoExtractor):
             'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
             'upload_date': unified_strdate(info['releaseDate'].split('.')[0]),
         }
+        if video_url.startswith('rtmp:'):
+            mobj = re.match(r'(?P<tc_url>rtmp://[^/]+/(?P<app>[a-z0-9/]+))/(?P<play_path>mp4:.*)', video_url)
+            ret.update({
+                'tc_url': mobj.group('tc_url'),
+                'play_path': mobj.group('play_path'),
+                'app': mobj.group('app'),
+                'no_resume': True,
+            })
+        return ret
 
 
 class NHLIE(NHLBaseInfoExtractor):
@@ -115,6 +124,18 @@ class NHLIE(NHLBaseInfoExtractor):
             'duration': 268,
             'upload_date': '20141122',
         }
+    }, {
+        'url': 'http://video.oilers.nhl.com/videocenter/console?id=691469&catid=4',
+        'info_dict': {
+            'id': '691469',
+            'ext': 'mp4',
+            'title': 'RAW | Craig MacTavish Full Press Conference',
+            'description': 'Oilers GM Craig MacTavish addresses the media at Rexall Place on Friday.',
+            'upload_date': '20141205',
+        },
+        'params': {
+            'skip_download': True,  # Requires rtmpdump
+        }
     }]
 
     def _real_extract(self, url):

From a745475808e125a590afb14df48c565309d3f75c Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 02:50:46 +0200
Subject: [PATCH 0518/1515] Ir90Tv Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ir90tv.py   | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/ir90tv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dfa781f8..ee05a6958 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -229,6 +229,7 @@ from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
+from .ir90tv import Ir90TvIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
new file mode 100644
index 000000000..5aa9d6ff4
--- /dev/null
+++ b/youtube_dl/extractor/ir90tv.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Ir90TvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
+    _TEST = {
+        'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'md5': '411dbd94891381960cb9e13daa47a869',
+        'info_dict': {
+            'id': '95719',
+            'ext': 'mp4',
+            'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # TODO more code goes here, for example ...
+        title = self._html_search_regex(
+            r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
+
+        video_url = self._search_regex(
+            r'<source[^>]+src="([^"]+)"', webpage, 'video url')
+
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
+        print thumbnail
+
+
+        return {
+            'url': video_url,
+            'id': video_id,
+            'title': title,
+            'video_url' : video_url,
+            'thumbnail' : thumbnail,
+        }
\ No newline at end of file

From 54b31d149e7be08eb7be9981a9eec398d11f17ef Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 02:55:01 +0200
Subject: [PATCH 0519/1515] Ir90Tv Add new extractor

---
 youtube_dl/extractor/ir90tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 5aa9d6ff4..3a3cb4887 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -38,4 +38,4 @@ class Ir90TvIE(InfoExtractor):
             'title': title,
             'video_url' : video_url,
             'thumbnail' : thumbnail,
-        }
\ No newline at end of file
+        }

From a650110ba762b2658c64392317c1afd2a284dd3d Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 04:32:08 +0200
Subject: [PATCH 0520/1515] remove print

---
 youtube_dl/extractor/ir90tv.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 3a3cb4887..b79529b1b 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -21,7 +21,6 @@ class Ir90TvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # TODO more code goes here, for example ...
         title = self._html_search_regex(
             r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
 
@@ -29,8 +28,6 @@ class Ir90TvIE(InfoExtractor):
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
         thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
-        print thumbnail
-
 
         return {
             'url': video_url,

From 541168039d8f3e7680a15cc366fcc94335308d81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 8 May 2015 11:01:24 +0200
Subject: [PATCH 0521/1515] [utils] get_exe_version: encode executable name
 (fixes #5647)

It failed in python 2.x when $PATH contains a directory with non-ascii characters.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1013f7c18..de09b53b2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1380,7 +1380,7 @@ def get_exe_version(exe, args=['--version'],
     or False if the executable is not present """
     try:
         out, _ = subprocess.Popen(
-            [exe] + args,
+            [encodeArgument(exe)] + args,
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
     except OSError:
         return False

From 249962ffa2b155795ebfe0a267eb025cd5e30c56 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 16:56:15 +0100
Subject: [PATCH 0522/1515] [bet] Use unique part of xml url as the video id
 and fix tests (closes #5642)

The guid changes often.
---
 youtube_dl/extractor/bet.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index d2abd4d77..26b934543 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,11 +16,11 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '740ab250-bb94-4a8a-8787-fe0de7c74471',
+                'id': 'news/national/2014/a-conversation-with-president-obama',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
-                'title': 'BET News Presents: A Conversation With President Obama',
-                'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
+                'title': 'A Conversation With President Obama',
+                'description': 'md5:699d0652a350cf3e491cd15cc745b5da',
                 'duration': 1534,
                 'timestamp': 1418075340,
                 'upload_date': '20141208',
@@ -35,7 +35,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d',
+                'id': 'news/national/2014/justice-for-ferguson-a-community-reacts',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
@@ -61,6 +61,9 @@ class BetIE(InfoExtractor):
             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
             webpage, 'media URL'))
 
+        video_id = self._search_regex(
+            r'/video/(.*)/_jcr_content/', media_url, 'video id')
+
         mrss = self._download_xml(media_url, display_id)
 
         item = mrss.find('./channel/item')
@@ -75,8 +78,6 @@ class BetIE(InfoExtractor):
         description = xpath_text(
             item, './description', 'description', fatal=False)
 
-        video_id = xpath_text(item, './guid', 'video id', fatal=False)
-
         timestamp = parse_iso8601(xpath_text(
             item, xpath_with_ns('./dc:date', NS_MAP),
             'upload date', fatal=False))

From 43837189c18af635cfb1cd8fe503265b4b218c32 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 8 May 2015 10:40:25 +0100
Subject: [PATCH 0523/1515] Fix URL template extraction for netzkino. Fixes
 #5614

---
 youtube_dl/extractor/netzkino.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
index bc17e20aa..0d165a82a 100644
--- a/youtube_dl/extractor/netzkino.py
+++ b/youtube_dl/extractor/netzkino.py
@@ -49,7 +49,7 @@ class NetzkinoIE(InfoExtractor):
             'http://www.netzkino.de/beta/dist/production.min.js', video_id,
             note='Downloading player code')
         avo_js = self._search_regex(
-            r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})',
+            r'var urlTemplate=(\{.*?"\})',
             production_js, 'URL templates')
         templates = self._parse_json(
             avo_js, video_id, transform_source=js_to_json)

From d1feb308116f57ceae3888db5e1b93394300f564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 20:07:53 +0600
Subject: [PATCH 0524/1515] [mlb] Fallback to extracting video id from webpage
 for all URLs that does not contain it explicitly (Closes #5630)

---
 youtube_dl/extractor/mlb.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index ee9ff73bf..109eecefd 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,21 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/(?:embed|m-internal-embed)\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/
+                        (?:
+                            (?:
+                                (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
+                                (?:
+                                    shared/video/embed/(?:embed|m-internal-embed)\.html|
+                                    [^/]+/video/play\.jsp
+                                )\?.*?\bcontent_id=
+                            )
+                            (?P<id>n?\d+)|
+                            (?P<path>.+?)
+                        )
+                    '''
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -95,6 +109,12 @@ class MLBIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        if not video_id:
+            video_path = mobj.group('path')
+            webpage = self._download_webpage(url, video_path)
+            video_id = self._search_regex(
+                r'data-videoid="(\d+)"', webpage, 'video id')
+
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'
             % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id)

From 4e6e9d21bd62c4e2ab2576347e066891092a5783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 21:48:47 +0600
Subject: [PATCH 0525/1515] [mlb] Improve _VALID_URL

---
 youtube_dl/extractor/mlb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 109eecefd..4e054fb53 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -22,7 +22,7 @@ class MLBIE(InfoExtractor):
                                 )\?.*?\bcontent_id=
                             )
                             (?P<id>n?\d+)|
-                            (?P<path>.+?)
+                            (?:[^/]+/)*(?P<path>[^/]+)
                         )
                     '''
     _TESTS = [

From 34e7dc81a94d39d48c5b4aac8cddcca46edba94d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:03:03 +0600
Subject: [PATCH 0526/1515] [vgtv] Add support for generic bt.no URLs (#5620)

---
 youtube_dl/extractor/vgtv.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 69dc9a759..b0f0b3bc2 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -8,7 +8,8 @@ from ..utils import float_or_none
 
 
 class VGTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/[^/]+/(?P<id>[0-9]+)'
+    IE_DESC = 'VGTV and BTTV'
+    _VALID_URL = r'http://(?:www\.)?(?P<host>vgtv|bt)\.no/(?:(?:tv/)?#!/(?:video|live)/(?P<id>[0-9]+)|(?:[^/]+/)*(?P<path>[^/]+))'
     _TESTS = [
         {
             # streamType: vod
@@ -64,12 +65,25 @@ class VGTVIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+
+        HOST_WEBSITES = {
+            'vgtv': 'vgtv',
+            'bt': 'bttv',
+        }
+
         data = self._download_json(
-            'http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website' % video_id,
+            'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
+            % (host, video_id, HOST_WEBSITES[host]),
             video_id, 'Downloading media JSON')
 
         streams = data['streamUrls']

From 0ceab8474924c4e7a6e28497c8da40cc5002c8d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:18:43 +0600
Subject: [PATCH 0527/1515] [vgtv] Add support for bt.no articles (#5620)

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/vgtv.py     | 39 +++++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dfa781f8..587a45940 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -587,7 +587,10 @@ from .veoh import VeohIE
 from .vessel import VesselIE
 from .vesti import VestiIE
 from .vevo import VevoIE
-from .vgtv import VGTVIE
+from .vgtv import (
+    BTArticleIE,
+    VGTVIE,
+)
 from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index b0f0b3bc2..ad07e54c9 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -9,7 +9,18 @@ from ..utils import float_or_none
 
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV and BTTV'
-    _VALID_URL = r'http://(?:www\.)?(?P<host>vgtv|bt)\.no/(?:(?:tv/)?#!/(?:video|live)/(?P<id>[0-9]+)|(?:[^/]+/)*(?P<path>[^/]+))'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        vgtv:|
+                        http://(?:www\.)?
+                    )
+                    (?P<host>vgtv|bt)
+                    (?:
+                        :|
+                        \.no/(?:tv/)?#!/(?:video|live)/
+                    )
+                    (?P<id>[0-9]+)
+                    '''
     _TESTS = [
         {
             # streamType: vod
@@ -129,3 +140,29 @@ class VGTVIE(InfoExtractor):
             'view_count': data['displays'],
             'formats': formats,
         }
+
+
+class BTArticleIE(InfoExtractor):
+    IE_DESC = 'Bergens Tidende'
+    _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _TEST = {
+        'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
+        'md5': 'd055e8ee918ef2844745fcfd1a4175fb',
+        'info_dict': {
+            'id': '23199',
+            'ext': 'mp4',
+            'title': 'Alrekstad internat',
+            'description': 'md5:dc81a9056c874fedb62fc48a300dac58',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 191,
+            'timestamp': 1289991323,
+            'upload_date': '20101117',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, self._match_id(url))
+        video_id = self._search_regex(
+            r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
+        return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')

From 2c0c9dc46cda490137b6788d6d66f31ca092f58f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:50:01 +0600
Subject: [PATCH 0528/1515] [xstream] Move xstream to separate extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/xstream.py  | 115 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/xstream.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 587a45940..5cc35c8eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -653,6 +653,7 @@ from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
+from .xstream import XstreamIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
new file mode 100644
index 000000000..71584c291
--- /dev/null
+++ b/youtube_dl/extractor/xstream.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    find_xpath_attr,
+)
+
+
+class XstreamIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        xstream:|
+                        https?://frontend\.xstream\.(?:dk|net)/
+                    )
+                    (?P<partner_id>[^/]+)
+                    (?:
+                        :|
+                        /feed/video/\?.*?\bid=
+                    )
+                    (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'http://frontend.xstream.dk/btno/feed/video/?platform=web&id=86588',
+        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
+        'info_dict': {
+            'id': '86588',
+            'ext': 'mov',
+            'title': 'Otto Wollertsen',
+            'description': 'Vestlendingen Otto Fredrik Wollertsen',
+            'timestamp': 1430473209,
+            'upload_date': '20150501',
+        },
+    }, {
+        'url': 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=21039',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+
+        data = self._download_xml(
+            'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s'
+            % (partner_id, video_id),
+            video_id)
+
+        NS_MAP = {
+            'atom': 'http://www.w3.org/2005/Atom',
+            'xt': 'http://xstream.dk/',
+            'media': 'http://search.yahoo.com/mrss/',
+        }
+
+        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
+
+        title = xpath_text(
+            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
+        description = xpath_text(
+            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
+        timestamp = parse_iso8601(xpath_text(
+            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
+
+        formats = []
+        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
+        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
+            media_url = media_content.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media_content.get('bitrate'))
+            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
+            if mobj:
+                formats.append({
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:%s' % mobj.group('playpath'),
+                    'app': mobj.group('app'),
+                    'ext': 'flv',
+                    'tbr': tbr,
+                    'format_id': 'rtmp-%d' % tbr,
+                })
+            else:
+                formats.append({
+                    'url': media_url,
+                    'tbr': tbr,
+                })
+        self._sort_formats(formats)
+
+        link = find_xpath_attr(
+            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
+        if link is not None:
+            formats.append({
+                'url': link.get('href'),
+                'format_id': link.get('rel'),
+            })
+
+        thumbnails = [{
+            'url': splash.get('url'),
+            'width': int_or_none(splash.get('width')),
+            'height': int_or_none(splash.get('height')),
+        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From cbe443362f91ab111e2a01fe8246e17a98668f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:52:20 +0600
Subject: [PATCH 0529/1515] [aftenposten] Implement in terms of xtream
 extractor

---
 youtube_dl/extractor/aftenposten.py | 77 +----------------------------
 1 file changed, 1 insertion(+), 76 deletions(-)

diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
index e15c015fb..0c00acfb5 100644
--- a/youtube_dl/extractor/aftenposten.py
+++ b/youtube_dl/extractor/aftenposten.py
@@ -1,21 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_iso8601,
-    xpath_with_ns,
-    xpath_text,
-    find_xpath_attr,
-)
 
 
 class AftenpostenIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
-
     _TEST = {
         'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
         'md5': 'fd828cd29774a729bf4d4425fe192972',
@@ -30,69 +20,4 @@ class AftenpostenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        data = self._download_xml(
-            'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)
-
-        NS_MAP = {
-            'atom': 'http://www.w3.org/2005/Atom',
-            'xt': 'http://xstream.dk/',
-            'media': 'http://search.yahoo.com/mrss/',
-        }
-
-        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
-
-        title = xpath_text(
-            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
-        description = xpath_text(
-            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
-        timestamp = parse_iso8601(xpath_text(
-            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
-
-        formats = []
-        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
-        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
-            media_url = media_content.get('url')
-            if not media_url:
-                continue
-            tbr = int_or_none(media_content.get('bitrate'))
-            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
-            if mobj:
-                formats.append({
-                    'url': mobj.group('url'),
-                    'play_path': 'mp4:%s' % mobj.group('playpath'),
-                    'app': mobj.group('app'),
-                    'ext': 'flv',
-                    'tbr': tbr,
-                    'format_id': 'rtmp-%d' % tbr,
-                })
-            else:
-                formats.append({
-                    'url': media_url,
-                    'tbr': tbr,
-                })
-        self._sort_formats(formats)
-
-        link = find_xpath_attr(
-            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
-        if link is not None:
-            formats.append({
-                'url': link.get('href'),
-                'format_id': link.get('rel'),
-            })
-
-        thumbnails = [{
-            'url': splash.get('url'),
-            'width': int_or_none(splash.get('width')),
-            'height': int_or_none(splash.get('height')),
-        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'formats': formats,
-            'thumbnails': thumbnails,
-        }
+        return self.url_result('xstream:ap:%s' % self._match_id(url), 'Xstream')

From fe373287ebdda002ed84dca1d8b9d6f8a5686138 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:59:50 +0600
Subject: [PATCH 0530/1515] [vgtv] Add support for bt vestlendingen (Closes
 #5620)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vgtv.py     | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5cc35c8eb..96cf28efe 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -589,6 +589,7 @@ from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import (
     BTArticleIE,
+    BTVestlendingenIE,
     VGTVIE,
 )
 from .vh1 import VH1IE
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index ad07e54c9..db7a4bdb1 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -143,7 +143,8 @@ class VGTVIE(InfoExtractor):
 
 
 class BTArticleIE(InfoExtractor):
-    IE_DESC = 'Bergens Tidende'
+    IE_NAME = 'bt:article'
+    IE_DESC = 'Bergens Tidende Articles'
     _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
@@ -166,3 +167,26 @@ class BTArticleIE(InfoExtractor):
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
         return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')
+
+
+class BTVestlendingenIE(InfoExtractor):
+    IE_NAME = 'bt:vestlendingen'
+    IE_DESC = 'Bergens Tidende - Vestlendingen'
+    _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
+        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
+        'info_dict': {
+            'id': '86588',
+            'ext': 'mov',
+            'title': 'Otto Wollertsen',
+            'description': 'Vestlendingen Otto Fredrik Wollertsen',
+            'timestamp': 1430473209,
+            'upload_date': '20150501',
+        },
+    }
+
+    def _real_extract(self, url):
+        return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
+
+

From 4384cf9e7d59492141ebd45f77830238097c695c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:04:27 +0600
Subject: [PATCH 0531/1515] [extractor/__init__] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 96cf28efe..7f0070784 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -655,9 +655,9 @@ from .xhamster import XHamsterIE
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
-from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
+from .xvideos import XVideosIE
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,

From bb03fdae0d4da9c591c2967044c5e30bf797c22a Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:19:57 +0600
Subject: [PATCH 0532/1515] [README.md] Clarify format selection when streaming
 to stdout

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9aeb114f3..b6e9429df 100644
--- a/README.md
+++ b/README.md
@@ -269,7 +269,7 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded an muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 

From 50b901306406d5c37f31880860e2a4dbb5e0a165 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:21:23 +0600
Subject: [PATCH 0533/1515] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b6e9429df..3d9436456 100644
--- a/README.md
+++ b/README.md
@@ -269,7 +269,7 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded an muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 

From 79998cd5afb2e16fe14cebdbec81b21c45c24c32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:12:42 +0600
Subject: [PATCH 0534/1515] [svtplay] Generalize svt extractors and add svt.se
 extractor

---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/svtplay.py  | 98 ++++++++++++++++++++------------
 2 files changed, 67 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f0070784..79236c6c1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -505,7 +505,10 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
-from .svtplay import SVTPlayIE
+from .svtplay import (
+    SVTIE,
+    SVTPlayIE,
+)
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index 433dfd1cb..732f02048 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -9,41 +9,9 @@ from ..utils import (
 )
 
 
-class SVTPlayIE(InfoExtractor):
-    IE_DESC = 'SVT Play and Öppet arkiv'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'ade3def0643fa1c40587a422f98edfd9',
-        'info_dict': {
-            'id': '2609989',
-            'ext': 'flv',
-            'title': 'SM veckan vinter, Örebro - Rally, final',
-            'duration': 4500,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-    }, {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
-
-        info = self._download_json(
-            'http://www.%s.se/video/%s?output=json' % (host, video_id), video_id)
+class SVTBaseIE(InfoExtractor):
+    def _extract_video(self, url, video_id):
+        info = self._download_json(url, video_id)
 
         title = info['context']['title']
         thumbnail = info['context'].get('thumbnailImage')
@@ -80,3 +48,63 @@ class SVTPlayIE(InfoExtractor):
             'duration': duration,
             'age_limit': age_limit,
         }
+
+
+class SVTIE(SVTBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
+        'md5': '9648197555fc1b49e3dc22db4af51d46',
+        'info_dict': {
+            'id': '2900353',
+            'ext': 'flv',
+            'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+            'duration': 27,
+            'age_limit': 0,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        widget_id = mobj.group('widget_id')
+        article_id = mobj.group('id')
+        return self._extract_video(
+            'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
+            article_id)
+
+
+class SVTPlayIE(SVTBaseIE):
+    IE_DESC = 'SVT Play and Öppet arkiv'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
+        'md5': 'ade3def0643fa1c40587a422f98edfd9',
+        'info_dict': {
+            'id': '2609989',
+            'ext': 'flv',
+            'title': 'SM veckan vinter, Örebro - Rally, final',
+            'duration': 4500,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'flv',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+        'skip': 'Only works from Sweden',
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+        return self._extract_video(
+            'http://www.%s.se/video/%s?output=json' % (host, video_id),
+            video_id)

From 322915014f0378e2675a2a17cd67fe89a6e6a7d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:13:40 +0600
Subject: [PATCH 0535/1515] [svtplay] Rename to svt

---
 youtube_dl/extractor/__init__.py            | 2 +-
 youtube_dl/extractor/{svtplay.py => svt.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{svtplay.py => svt.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79236c6c1..0a18dba5c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -505,7 +505,7 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
-from .svtplay import (
+from .svt import (
     SVTIE,
     SVTPlayIE,
 )
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svt.py
similarity index 100%
rename from youtube_dl/extractor/svtplay.py
rename to youtube_dl/extractor/svt.py

From bab19a8e91153705d6600fe1d1a0b0aa0bf93bb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:23:35 +0600
Subject: [PATCH 0536/1515] [extractor/generic] Add support for svt embeds
 (Closes #5622)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 youtube_dl/extractor/svt.py     | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cd7c47d6d..046bcb0f0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,6 +37,7 @@ from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
+from .svt import SVTIE
 
 
 class GenericIE(InfoExtractor):
@@ -1091,6 +1092,11 @@ class GenericIE(InfoExtractor):
         if bliptv_url:
             return self.url_result(bliptv_url, 'BlipTV')
 
+        # Look for SVT player
+        svt_url = SVTIE._extract_url(webpage)
+        if svt_url:
+            return self.url_result(svt_url, 'SVT')
+
         # Look for embedded condenast player
         matches = re.findall(
             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 732f02048..fc20f664b 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -64,6 +64,13 @@ class SVTIE(SVTBaseIE):
         },
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'(?:<iframe src|href)="(?P<url>%s[^"]*)"' % SVTIE._VALID_URL, webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         widget_id = mobj.group('widget_id')

From dc455a5f88d6bb9bd5a44352e2b67ef7dd5fd683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:27:37 +0600
Subject: [PATCH 0537/1515] [extractor/generic] Add test for svt embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 046bcb0f0..7c38bce7c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -659,6 +659,17 @@ class GenericIE(InfoExtractor):
                 'title': 'Facebook Creates "On This Day" | Crunch Report',
             },
         },
+        # SVT embed
+        {
+            'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
+            'info_dict': {
+                'id': '2900353',
+                'ext': 'flv',
+                'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+                'duration': 27,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',

From de765f6c3188802bb2dea704a645f539fa61c8aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 02:15:51 +0600
Subject: [PATCH 0538/1515] [foxsports] Support some more URLs (#5611)

---
 youtube_dl/extractor/foxsports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index 363866b64..df7665176 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -5,7 +5,7 @@ from ..utils import smuggle_url
 
 
 class FoxSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/video\?vid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.foxsports.com/video?vid=432609859715',

From 3dbec410a0e195036025aa3a3792932783f371d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 13:19:54 +0800
Subject: [PATCH 0539/1515] [sohu] Enhance error handling

---
 youtube_dl/extractor/sohu.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index f8a4840f7..13b9e9133 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import sanitize_url_path_consecutive_slashes
+from ..utils import (
+    sanitize_url_path_consecutive_slashes,
+    ExtractorError,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -117,6 +120,15 @@ class SohuIE(InfoExtractor):
             r'var vid ?= ?["\'](\d+)["\']',
             webpage, 'video path')
         vid_data = _fetch_data(vid, mytv)
+        if vid_data['play'] != 1:
+            if vid_data.get('status') == 12:
+                raise ExtractorError(
+                    'Sohu said: There\'s something wrong in the video.',
+                    expected=True)
+            else:
+                raise ExtractorError(
+                    'Sohu said: The video is only licensed to users in Mainland China.',
+                    expected=True)
 
         formats_json = {}
         for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):

From 32060c6d6b618fa858b2ce43db34d02fd43bc542 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 13:54:28 +0800
Subject: [PATCH 0540/1515] [sohu] Update extractor

The original extraction logic always fails for all test videos
---
 youtube_dl/extractor/sohu.py | 44 ++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 13b9e9133..eab4adfca 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,10 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import (
-    sanitize_url_path_consecutive_slashes,
-    ExtractorError,
-)
+from ..utils import ExtractorError
 
 
 class SohuIE(InfoExtractor):
@@ -31,7 +28,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': '699060e75cf58858dd47fb9c03c42cfb',
+        'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -39,7 +36,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '9bf34be48f2f4dadcb226c74127e203c',
+        'md5': '49308ff6dafde5ece51137d04aec311e',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -53,7 +50,7 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
+            'md5': '492923eac023ba2f13ff69617c32754a',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -61,7 +58,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
+            'md5': 'de604848c0e8e9c4a4dde7e1347c0637',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -69,7 +66,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '8407e634175fdac706766481b9443450',
+            'md5': '93584716ee0657c0b205b8aa3d27aa13',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',
@@ -144,24 +141,21 @@ class SohuIE(InfoExtractor):
         for i in range(part_count):
             formats = []
             for format_id, format_data in formats_json.items():
-                allot = format_data['allot']
-                prot = format_data['prot']
-
                 data = format_data['data']
-                clips_url = data['clipsURL']
-                su = data['su']
 
-                part_str = self._download_webpage(
-                    'http://%s/?prot=%s&file=%s&new=%s' %
-                    (allot, prot, clips_url[i], su[i]),
-                    video_id,
-                    'Downloading %s video URL part %d of %d'
-                    % (format_id, i + 1, part_count))
-
-                part_info = part_str.split('|')
-
-                video_url = sanitize_url_path_consecutive_slashes(
-                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
+                # URLs starts with http://newflv.sohu.ccgslb.net/ is not usable
+                # so retry until got a working URL
+                video_url = 'newflv.sohu.ccgslb.net'
+                retries = 0
+                while 'newflv.sohu.ccgslb.net' in video_url and retries < 5:
+                    download_note = 'Download information from CDN gateway for format ' + format_id
+                    if retries > 0:
+                        download_note += ' (retry #%d)' % retries
+                    retries += 1
+                    cdn_info = self._download_json(
+                        'http://data.vod.itc.cn/cdnList?new=' + data['su'][i],
+                        video_id, download_note)
+                    video_url = cdn_info['url']
 
                 formats.append({
                     'url': video_url,

From 6d14d08e062ff3d6e0fd17f04cb341099097902c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 17:36:07 +0800
Subject: [PATCH 0541/1515] [yam] Fix title and uploader id

---
 youtube_dl/extractor/yam.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 19f8762ae..9d851bae3 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     month_by_abbreviation,
     ExtractorError,
+    get_element_by_attribute,
 )
 
 
@@ -23,6 +24,7 @@ class YamIE(InfoExtractor):
             'id': '2283921',
             'ext': 'mp3',
             'title': '發現 - 趙薇 京華煙雲主題曲',
+            'description': '發現 - 趙薇 京華煙雲主題曲',
             'uploader_id': 'princekt',
             'upload_date': '20080807',
             'duration': 313.0,
@@ -55,6 +57,17 @@ class YamIE(InfoExtractor):
             'ext': 'mp4',
         },
         'skip': 'invalid YouTube URL',
+    }, {
+        'url': 'http://mymedia.yam.com/m/2373534',
+        'md5': '7ff74b91b7a817269d83796f8c5890b1',
+        'info_dict': {
+            'id': '2373534',
+            'ext': 'mp3',
+            'title': '林俊傑&蔡卓妍-小酒窩',
+            'description': 'md5:904003395a0fcce6cfb25028ff468420',
+            'upload_date': '20080928',
+            'uploader_id': 'onliner2',
+        }
     }]
 
     def _real_extract(self, url):
@@ -75,15 +88,19 @@ class YamIE(InfoExtractor):
         if youtube_url:
             return self.url_result(youtube_url, 'Youtube')
 
+        title = self._html_search_regex(
+            r'<h1[^>]+class="heading"[^>]*>\s*(.+)\s*</h1>', page, 'title')
+
         api_page = self._download_webpage(
             'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
             note='Downloading API page')
         api_result_obj = compat_urlparse.parse_qs(api_page)
 
+        info_table = get_element_by_attribute('class', 'info', page)
         uploader_id = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"',
-            page, 'uploader id', fatal=False)
-        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z0-9]+)"',
+            info_table, 'uploader id', fatal=False)
+        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})\s+' +
                          r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
         if mobj:
             upload_date = '%s%02d%02d' % (
@@ -97,7 +114,8 @@ class YamIE(InfoExtractor):
         return {
             'id': video_id,
             'url': api_result_obj['mp3file'][0],
-            'title': self._html_search_meta('description', page),
+            'title': title,
+            'description': self._html_search_meta('description', page),
             'duration': duration,
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From d39e0f05db226ef5691f5730d40da796aec6bac6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 17:37:39 +0800
Subject: [PATCH 0542/1515] [utils] Remove
 sanitize_url_path_consecutive_slashes()

This function is used only in SohuIE, which is updated to use a new
extraction logic.
---
 test/test_utils.py  | 21 ---------------------
 youtube_dl/utils.py |  7 -------
 2 files changed, 28 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 032d3656a..86b110a7d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -40,7 +40,6 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
-    sanitize_url_path_consecutive_slashes,
     prepend_extension,
     replace_extension,
     shell_quote,
@@ -176,26 +175,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
-    def test_sanitize_url_path_consecutive_slashes(self):
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/foo//bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname//foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname//'),
-            'http://hostname/')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/'),
-            'http://hostname/')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
-            'http://hostname/abc/')
-
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
         self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index de09b53b2..d73efcf25 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -327,13 +327,6 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
-def sanitize_url_path_consecutive_slashes(url):
-    """Collapses consecutive slashes in URLs' path"""
-    parsed_url = list(compat_urlparse.urlparse(url))
-    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
-    return compat_urlparse.urlunparse(parsed_url)
-
-
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From 5c0b2c16a80c509dbcee48f48da3de0bf9912cda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 May 2015 12:34:45 +0200
Subject: [PATCH 0543/1515] [vgtv] Escape '#' in _VALID_URL and remove empty
 newlines at the end

In verbose mode, '#' is interpreted as the start of a comment.
---
 youtube_dl/extractor/vgtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index db7a4bdb1..eb2652fb1 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
                     (?P<host>vgtv|bt)
                     (?:
                         :|
-                        \.no/(?:tv/)?#!/(?:video|live)/
+                        \.no/(?:tv/)?\#!/(?:video|live)/
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -188,5 +188,3 @@ class BTVestlendingenIE(InfoExtractor):
 
     def _real_extract(self, url):
         return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
-
-

From 3b5f65a64c06859cdee0b93f319c80d5c116cedd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 May 2015 12:41:56 +0200
Subject: [PATCH 0544/1515] [mlb] Fix extraction of articles

And move test from generic, since it's directly handled by MLBIE
---
 youtube_dl/extractor/generic.py | 13 -------------
 youtube_dl/extractor/mlb.py     | 14 +++++++++++++-
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7c38bce7c..3d756e848 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -414,19 +414,6 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
-        # MLB articles
-        {
-            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
-            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
-            'info_dict': {
-                'id': '75609783',
-                'ext': 'mp4',
-                'title': 'Must C: Pillar climbs for catch',
-                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
-                'timestamp': 1429124820,
-                'upload_date': '20150415',
-            }
-        },
         # Wistia embed
         {
             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 4e054fb53..40c9ecb35 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -82,6 +82,18 @@ class MLBIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        {
+            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
+            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
+            'info_dict': {
+                'id': '75609783',
+                'ext': 'mp4',
+                'title': 'Must C: Pillar climbs for catch',
+                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
+                'timestamp': 1429124820,
+                'upload_date': '20150415',
+            }
+        },
         {
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
@@ -113,7 +125,7 @@ class MLBIE(InfoExtractor):
             video_path = mobj.group('path')
             webpage = self._download_webpage(url, video_path)
             video_id = self._search_regex(
-                r'data-videoid="(\d+)"', webpage, 'video id')
+                r'data-video-?id="(\d+)"', webpage, 'video id')
 
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'

From d592b42f5c37c4b0d9f1587dd76f225b0287959f Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Sat, 9 May 2015 15:26:00 +0100
Subject: [PATCH 0545/1515] Updated two tests for BRIE

---
 youtube_dl/extractor/br.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 45ba51732..04a3ecd17 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -16,27 +16,27 @@ class BRIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.br.de/mediathek/video/sendungen/heimatsound/heimatsound-festival-2014-trailer-100.html',
-            'md5': '93556dd2bcb2948d9259f8670c516d59',
+            'url': 'http://www.br.de/mediathek/video/sendungen/abendschau/betriebliche-altersvorsorge-104.html',
+            'md5': '83a0477cf0b8451027eb566d88b51106',
             'info_dict': {
-                'id': '25e279aa-1ffd-40fd-9955-5325bd48a53a',
+                'id': '48f656ef-287e-486f-be86-459122db22cc',
                 'ext': 'mp4',
-                'title': 'Wenn das Traditions-Theater wackelt',
-                'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt',
-                'duration': 34,
-                'uploader': 'BR',
-                'upload_date': '20140802',
+                'title': 'Die böse Überraschung',
+                'description': 'Betriebliche Altersvorsorge: Die böse Überraschung',
+                'duration': 180,
+                'uploader': 'Reinhard Weber',
+                'upload_date': '20150422',
             }
         },
         {
-            'url': 'http://www.br.de/nachrichten/schaeuble-haushaltsentwurf-bundestag-100.html',
-            'md5': '3db0df1a9a9cd9fa0c70e6ea8aa8e820',
+            'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
+            'md5': 'a44396d73ab6a68a69a568fae10705bb',
             'info_dict': {
-                'id': 'c6aae3de-2cf9-43f2-957f-f17fef9afaab',
-                'ext': 'aac',
-                'title': '"Keine neuen Schulden im nächsten Jahr"',
-                'description': 'Haushaltsentwurf: "Keine neuen Schulden im nächsten Jahr"',
-                'duration': 64,
+                'id': 'a4b83e34-123d-4b81-9f4e-c0d3121a4e05',
+                'ext': 'mp4',
+                'title': 'Manfred Schreiber ist tot',
+                'description': 'Abendschau kompakt: Manfred Schreiber ist tot',
+                'duration': 26,
             }
         },
         {

From 0892090a56e04726175c247d13ecce7f6c9cb839 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Sat, 9 May 2015 16:02:07 +0100
Subject: [PATCH 0546/1515] Added audio test for BRIE

---
 youtube_dl/extractor/br.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 04a3ecd17..66e394e10 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -39,6 +39,17 @@ class BRIE(InfoExtractor):
                 'duration': 26,
             }
         },
+        {
+            'url': 'http://www.br.de/radio/br-klassik/sendungen/allegro/premiere-urauffuehrung-the-land-2015-dance-festival-muenchen-100.html',
+            'md5': '8b5b27c0b090f3b35eac4ab3f7a73d3d',
+            'info_dict': {
+                'id': '74c603c9-26d3-48bb-b85b-079aeed66e0b',
+                'ext': 'aac',
+                'title': 'Kurzweilig und sehr bewegend',
+                'description': '"The Land" von Peeping Tom: Kurzweilig und sehr bewegend',
+                'duration': 296,
+            }
+        },
         {
             'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',
             'md5': 'dbab0aef2e047060ea7a21fc1ce1078a',

From 32fffff2ccc044c639c8723281981aa347423762 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 21:19:09 +0600
Subject: [PATCH 0547/1515] [eroprofile] Fix video URL extraction (Closes
 #5657)

---
 youtube_dl/extractor/eroprofile.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 0cbca90b0..316033cf1 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -4,7 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    unescapeHTML
+)
 
 
 class EroProfileIE(InfoExtractor):
@@ -75,8 +78,8 @@ class EroProfileIE(InfoExtractor):
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)
 
-        video_url = self._search_regex(
-            r'<source src="([^"]+)', webpage, 'video url')
+        video_url = unescapeHTML(self._search_regex(
+            r'<source src="([^"]+)', webpage, 'video url'))
         title = self._html_search_regex(
             r'Title:</th><td>([^<]+)</td>', webpage, 'title')
         thumbnail = self._search_regex(

From f2e0056579ac507b776ce2c86b5281fc28bbc275 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 21:23:09 +0600
Subject: [PATCH 0548/1515] [vgtv] Avoid duplicate format_id

---
 youtube_dl/extractor/vgtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index eb2652fb1..e6ee1e471 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -103,11 +103,14 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4'))
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls'))
 
         hds_url = streams.get('hds')
         if hds_url:
-            formats.extend(self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id))
+            formats.extend(self._extract_f4m_formats(
+                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                video_id, f4m_id='hds'))
 
         mp4_url = streams.get('mp4')
         if mp4_url:

From 480065172d4c97f00973b3f0bf24cd1b8e567627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 00:26:42 +0600
Subject: [PATCH 0549/1515] [lifenews] Add support for video URLs (Closes
 #5660)

---
 youtube_dl/extractor/lifenews.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 081016b80..92031e843 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -14,7 +14,7 @@ from ..utils import (
 class LifeNewsIE(InfoExtractor):
     IE_NAME = 'lifenews'
     IE_DESC = 'LIFE | NEWS'
-    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?news/(?P<id>\d+)'
+    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://lifenews.ru/news/126342',
@@ -55,12 +55,15 @@ class LifeNewsIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        section = mobj.group('section')
 
-        webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
+        webpage = self._download_webpage(
+            'http://lifenews.ru/%s/%s' % (section, video_id),
+            video_id, 'Downloading page')
 
         videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
         iframe_link = self._html_search_regex(
-            '<iframe[^>]+src="([^"]+)', webpage, 'iframe link', default=None)
+            '<iframe[^>]+src=["\']([^"\']+)["\']', webpage, 'iframe link', default=None)
         if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 

From 057ebeaca3da40b901b2592e2302a0d4bbab48e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 00:27:49 +0600
Subject: [PATCH 0550/1515] [lifenews] Add test for #5660

---
 youtube_dl/extractor/lifenews.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 92031e843..7d5b8621b 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -50,6 +50,9 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150505',
             'uploader': 'embed.life.ru',
         }
+    }, {
+        'url': 'http://lifenews.ru/video/13035',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 754270313a2b337eda98fa95232bd1064e294173 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 01:03:26 +0600
Subject: [PATCH 0551/1515] [life:embed] Move to separated extractor and
 extract m3u8 formats

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/lifenews.py | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a18dba5c..f117578a2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -258,7 +258,10 @@ from .letv import (
     LetvPlaylistIE
 )
 from .libsyn import LibsynIE
-from .lifenews import LifeNewsIE
+from .lifenews import (
+    LifeNewsIE,
+    LifeEmbedIE,
+)
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 7d5b8621b..7f39fa4cf 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -4,7 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
+    determine_ext,
     int_or_none,
     unified_strdate,
     ExtractorError,
@@ -119,3 +121,48 @@ class LifeNewsIE(InfoExtractor):
             return make_entry(video_id, videos[0])
         else:
             return [make_entry(video_id, media, video_number + 1) for video_number, media in enumerate(videos)]
+
+
+class LifeEmbedIE(InfoExtractor):
+    IE_NAME = 'life:embed'
+    _VALID_URL = r'http://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
+
+    _TEST = {
+        'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
+        'md5': 'b889715c9e49cb1981281d0e5458fbbe',
+        'info_dict': {
+            'id': 'e50c2dec2867350528e2574c899b8291',
+            'ext': 'mp4',
+            'title': 'e50c2dec2867350528e2574c899b8291',
+            'thumbnail': 're:http://.*\.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for video_url in re.findall(r'"file"\s*:\s*"([^"]+)', webpage):
+            video_url = compat_urlparse.urljoin(url, video_url)
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='m3u8'))
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': ext,
+                    'preference': 1,
+                })
+
+        thumbnail = self._search_regex(
+            r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)
+
+        return {
+            'id': video_id,
+            'title': video_id,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 69fe3a5f0961c0ae602da531d2b0fb3f11b9d7c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 10 May 2015 01:05:24 +0200
Subject: [PATCH 0552/1515] release 2015.05.10

---
 docs/supportedsites.md | 9 ++++++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6d2e496a8..98b625380 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -64,6 +64,8 @@
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
  - **Brightcove**
+ - **bt:article**: Bergens Tidende Articles
+ - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
  - **BYUtv**
  - **Camdemy**
@@ -240,6 +242,7 @@
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
+ - **life:embed**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
@@ -328,6 +331,7 @@
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
+ - **NYTimesArticle**
  - **ocw.mit.edu**
  - **Odnoklassniki**
  - **OktoberfestTV**
@@ -434,6 +438,7 @@
  - **southpark.cc.com**
  - **southpark.cc.com:español**
  - **southpark.de**
+ - **southpark.nl**
  - **Space**
  - **SpankBang**
  - **Spankwire**
@@ -453,6 +458,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
+ - **SVT**
  - **SVTPlay**: SVT Play and Öppet arkiv
  - **SWRMediathek**
  - **Syfy**
@@ -529,7 +535,7 @@
  - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
- - **VGTV**
+ - **VGTV**: VGTV and BTTV
  - **vh1.com**
  - **Vice**
  - **Viddler**
@@ -587,6 +593,7 @@
  - **XHamster**
  - **XMinus**
  - **XNXX**
+ - **Xstream**
  - **XTube**
  - **XTubeUser**: XTube user profile
  - **Xuite**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a5a81bcd2..83c5a1659 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.04'
+__version__ = '2015.05.10'

From 3800b908b1976f242b41d5a2d114418559ce3b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 06:14:34 +0600
Subject: [PATCH 0553/1515] [mlb] Fix #5663

---
 youtube_dl/extractor/mlb.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 40c9ecb35..e242b897f 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -12,13 +12,13 @@ from ..utils import (
 class MLBIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/
+                        (?:[\da-z_-]+\.)*mlb\.com/
                         (?:
                             (?:
                                 (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
                                 (?:
                                     shared/video/embed/(?:embed|m-internal-embed)\.html|
-                                    [^/]+/video/play\.jsp
+                                    (?:[^/]+/)+(?:play|index)\.jsp|
                                 )\?.*?\bcontent_id=
                             )
                             (?P<id>n?\d+)|
@@ -114,6 +114,10 @@ class MLBIE(InfoExtractor):
             # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
             'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
             'only_matching': True,
+        },
+        {
+            'url': 'http://washington.nationals.mlb.com/mlb/gameday/index.jsp?c_id=was&gid=2015_05_09_atlmlb_wasmlb_1&lang=en&content_id=108309983&mode=video#',
+            'only_matching': True,
         }
     ]
 
@@ -125,7 +129,7 @@ class MLBIE(InfoExtractor):
             video_path = mobj.group('path')
             webpage = self._download_webpage(url, video_path)
             video_id = self._search_regex(
-                r'data-video-?id="(\d+)"', webpage, 'video id')
+                [r'data-video-?id="(\d+)"', r'content_id=(\d+)'], webpage, 'video id')
 
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'

From c6ddbdb66c5d6ead5e198013c54ef53d641063f1 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 12:30:07 +1200
Subject: [PATCH 0554/1515] [voicerepublic] Add new extractor

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/voicerepublic.py | 55 +++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/voicerepublic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f117578a2..5cb3c304d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -634,6 +634,7 @@ from .vk import (
     VKUserVideosIE,
 )
 from .vodlocker import VodlockerIE
+from .voicerepublic import VoiceRepublicIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vube import VubeIE
diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
new file mode 100644
index 000000000..1a90693cb
--- /dev/null
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_urllib_request,
+)
+
+
+class VoiceRepublicIE(InfoExtractor):
+    _VALID_URL = r'https?://voicerepublic\.com/talks/(?P<id>[0-9a-z-]+)'
+    _TEST = {
+        'url': 'https://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
+        'md5': '0554a24d1657915aa8e8f84e15dc9353',
+        'info_dict': {
+            'id': '2296',
+            'ext': 'm4a',
+            'title': 'Watching the Watchers: Building a Sousveillance State',
+            'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
+            'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'creator': 'M. C. McGrath',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        req = compat_urllib_request.Request(url)
+        # Older versions of Firefox get redirected to an "upgrade browser" page
+        req.add_header('User-Agent', 'youtube-dl')
+        webpage = self._download_webpage(req, display_id)
+        thumbnail = self._og_search_thumbnail(webpage)
+        video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
+
+        if '<div class=\'vr-player jp-jplayer\'' in webpage:
+            formats = [{
+                'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+                'ext': ext,
+                'format_id': ext,
+                'vcodec': 'none',
+            } for ext in ['m4a', 'mp3', 'ogg']]
+            self._sort_formats(formats)
+        else:
+            # Audio is still queued for processing
+            formats = []
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'url': self._og_search_url(webpage),
+            'thumbnail': thumbnail,
+            'description': self._og_search_description(webpage),
+            'creator': self._search_regex(r'<meta content=\'([^\']*?)\' name=\'author\'>', webpage, 'author', fatal=False),
+        }

From 95eb1adda8692a61db639fb21344ad22d1847044 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 08:54:50 +0600
Subject: [PATCH 0555/1515] [life:embed] Sort formats

---
 youtube_dl/extractor/lifenews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 7f39fa4cf..42cb6e35f 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -156,6 +156,7 @@ class LifeEmbedIE(InfoExtractor):
                     'format_id': ext,
                     'preference': 1,
                 })
+        self._sort_formats(formats)
 
         thumbnail = self._search_regex(
             r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)

From f900dc3fb9e17e399b0f33925ee239696cc46010 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:01:58 +1200
Subject: [PATCH 0556/1515] [voicerepublic] Extract author using
 _html_search_meta

---
 youtube_dl/extractor/voicerepublic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 1a90693cb..7d255d6fa 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -51,5 +51,5 @@ class VoiceRepublicIE(InfoExtractor):
             'url': self._og_search_url(webpage),
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'creator': self._search_regex(r'<meta content=\'([^\']*?)\' name=\'author\'>', webpage, 'author', fatal=False),
+            'creator': self._html_search_meta('author', webpage),
         }

From 03f760b1c0478c1f65cf6e978d7592be46873313 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:40:09 +1200
Subject: [PATCH 0557/1515] [voicerepublic] Remove creator field

---
 youtube_dl/extractor/voicerepublic.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 7d255d6fa..960974e16 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -19,7 +19,6 @@ class VoiceRepublicIE(InfoExtractor):
             'title': 'Watching the Watchers: Building a Sousveillance State',
             'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
             'description': 'md5:715ba964958afa2398df615809cfecb1',
-            'creator': 'M. C. McGrath',
         }
     }
 
@@ -51,5 +50,4 @@ class VoiceRepublicIE(InfoExtractor):
             'url': self._og_search_url(webpage),
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'creator': self._html_search_meta('author', webpage),
         }

From f03a8a3c4ec4dc95164c12181ffc1ddcb7583ef6 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:12:29 +1200
Subject: [PATCH 0558/1515] [voicerepublic] Raise ExtractorError if audio is
 still being processed

---
 youtube_dl/extractor/voicerepublic.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 960974e16..d3e35a815 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -2,10 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-from ..compat import (
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class VoiceRepublicIE(InfoExtractor):
@@ -31,17 +29,16 @@ class VoiceRepublicIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
 
-        if '<div class=\'vr-player jp-jplayer\'' in webpage:
-            formats = [{
-                'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
-                'ext': ext,
-                'format_id': ext,
-                'vcodec': 'none',
-            } for ext in ['m4a', 'mp3', 'ogg']]
-            self._sort_formats(formats)
-        else:
-            # Audio is still queued for processing
-            formats = []
+        if '<a>Queued for processing, please stand by...</a>' in webpage:
+            raise ExtractorError('Audio is still queued for processing')
+
+        formats = [{
+            'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+            'ext': ext,
+            'format_id': ext,
+            'vcodec': 'none',
+        } for ext in ['m4a', 'mp3', 'ogg']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 28ebef0b1b1b7b97137fbd8e093c09cb51954606 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 16:03:09 +1200
Subject: [PATCH 0559/1515] [voicerepublic] Detect list of available formats
 from the web page

---
 youtube_dl/extractor/voicerepublic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index d3e35a815..d150b5b5e 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import ExtractorError
@@ -32,12 +34,15 @@ class VoiceRepublicIE(InfoExtractor):
         if '<a>Queued for processing, please stand by...</a>' in webpage:
             raise ExtractorError('Audio is still queued for processing')
 
+        ext_matches = re.finditer(r'data-\w+=\'/vrmedia/\d+-clean\.(\w+)\'', webpage)
+        exts = [match.group(1) for match in ext_matches]
+
         formats = [{
             'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
             'ext': ext,
             'format_id': ext,
             'vcodec': 'none',
-        } for ext in ['m4a', 'mp3', 'ogg']]
+        } for ext in exts]
         self._sort_formats(formats)
 
         return {

From 1dcb52188d3709711b3ea5ae1ff6bdb985e79c62 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 16:38:26 +1200
Subject: [PATCH 0560/1515] [voicerepublic] Remove hardcoded paths to media
 files

---
 youtube_dl/extractor/voicerepublic.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index d150b5b5e..a3e40b940 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -34,15 +34,12 @@ class VoiceRepublicIE(InfoExtractor):
         if '<a>Queued for processing, please stand by...</a>' in webpage:
             raise ExtractorError('Audio is still queued for processing')
 
-        ext_matches = re.finditer(r'data-\w+=\'/vrmedia/\d+-clean\.(\w+)\'', webpage)
-        exts = [match.group(1) for match in ext_matches]
-
         formats = [{
-            'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+            'url': 'https://voicerepublic.com' + path,
             'ext': ext,
             'format_id': ext,
             'vcodec': 'none',
-        } for ext in exts]
+        } for ext, path in re.findall(r"data-([^=]+)='(/[^']+\.\1)'", webpage)]
         self._sort_formats(formats)
 
         return {

From a909e6ad43f9d9661691739a810d7b8853e17175 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 10 May 2015 15:27:55 +0800
Subject: [PATCH 0561/1515] [dailymotion] Patch upload_date detection.

(closes #5665)
---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index aa595af20..db10b8d00 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -52,6 +52,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'ext': 'mp4',
                 'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
+                'upload_date': '20150306',
             }
         },
         # Vevo video
@@ -106,9 +107,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         age_limit = self._rta_search(webpage)
 
         video_upload_date = None
-        mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
+        mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
         if mobj is not None:
-            video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
+            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
 
         embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
         embed_request = self._build_request(embed_url)

From 1934f3a0eaf16ae1d1644178b7128806b8629867 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 10 May 2015 18:22:07 +0800
Subject: [PATCH 0562/1515] [ndr] Extended to support n-joy.de as well (closes
 #4527)

According to http://en.wikipedia.org/wiki/N-Joy, n-joy.de is a service
hosted by NDR, so I put them together.
---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/ndr.py      | 90 ++++++++++++++++++++------------
 2 files changed, 61 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f117578a2..66adb4de7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -324,7 +324,10 @@ from .nbc import (
     NBCSportsIE,
     NBCSportsVPlayerIE,
 )
-from .ndr import NDRIE
+from .ndr import (
+    NDRIE,
+    NJoyIE,
+)
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index f49c66690..afb9eda27 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -8,41 +8,11 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     qualities,
+    parse_duration,
 )
 
 
-class NDRIE(InfoExtractor):
-    IE_NAME = 'ndr'
-    IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
-
-    _TESTS = [
-        {
-            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
-            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
-            'note': 'Video file',
-            'info_dict': {
-                'id': '25866',
-                'ext': 'mp4',
-                'title': 'Kartoffeltage in der Lewitz',
-                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
-                'duration': 166,
-            }
-        },
-        {
-            'url': 'http://www.ndr.de/info/audio51535.html',
-            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
-            'note': 'Audio file',
-            'info_dict': {
-                'id': '51535',
-                'ext': 'mp3',
-                'title': 'La Valette entgeht der Hinrichtung',
-                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
-                'duration': 884,
-            }
-        }
-    ]
-
+class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -54,7 +24,11 @@ class NDRIE(InfoExtractor):
         if description:
             description = description.strip()
 
-        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', fatal=False))
+        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', default=None))
+        if not duration:
+            duration = parse_duration(self._html_search_regex(
+                r'(<span class="min">\d+</span>:<span class="sec">\d+</span>)',
+                page, 'duration', default=None))
 
         formats = []
 
@@ -92,3 +66,53 @@ class NDRIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+
+class NDRIE(NDRBaseIE):
+    IE_NAME = 'ndr'
+    IE_DESC = 'NDR.de - Mediathek'
+    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
+
+    _TESTS = [
+        {
+            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
+            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
+            'note': 'Video file',
+            'info_dict': {
+                'id': '25866',
+                'ext': 'mp4',
+                'title': 'Kartoffeltage in der Lewitz',
+                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
+                'duration': 166,
+            }
+        },
+        {
+            'url': 'http://www.ndr.de/info/audio51535.html',
+            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+            'note': 'Audio file',
+            'info_dict': {
+                'id': '51535',
+                'ext': 'mp3',
+                'title': 'La Valette entgeht der Hinrichtung',
+                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
+                'duration': 884,
+            }
+        }
+    ]
+
+
+class NJoyIE(NDRBaseIE):
+    IE_NAME = 'N-JOY'
+    _VALID_URL = r'https?://www\.n-joy\.de/.+?(?P<id>\d+)\.html'
+
+    _TEST = {
+        'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': '2480',
+            'ext': 'mp4',
+            'title': 'Benaissa beim NDR Comedy Contest',
+            'description': 'Von seinem sehr "behaarten" Leben lässt sich Benaissa trotz aller Schwierigkeiten nicht unterkriegen.',
+            'duration': 654,
+        }
+    }

From 63cbd19f500eb4d90c1fc7c09f04de5df43a6a04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 10 May 2015 18:30:26 +0800
Subject: [PATCH 0563/1515] [ndr] Replace the 404 test case

---
 youtube_dl/extractor/ndr.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index afb9eda27..79a13958b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -84,7 +84,19 @@ class NDRIE(NDRBaseIE):
                 'title': 'Kartoffeltage in der Lewitz',
                 'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
                 'duration': 166,
-            }
+            },
+            'skip': '404 Not found',
+        },
+        {
+            'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
+            'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
+            'info_dict': {
+                'id': '988',
+                'ext': 'mp4',
+                'title': 'Party, Pötte und Parade',
+                'description': 'Hunderttausende feiern zwischen Speicherstadt und St. Pauli den 826. Hafengeburtstag. Die NDR Sondersendung zeigt die schönsten und spektakulärsten Bilder vom Auftakt.',
+                'duration': 3498,
+            },
         },
         {
             'url': 'http://www.ndr.de/info/audio51535.html',

From a6762c4a22325b5b69770de82df8725d2eb5c3df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 18:29:15 +0600
Subject: [PATCH 0564/1515] [voicerepublic] Make more robust and extract more
 metadata

---
 youtube_dl/extractor/voicerepublic.py | 95 ++++++++++++++++++++-------
 1 file changed, 71 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index a3e40b940..1106c655b 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -1,52 +1,99 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    int_or_none,
+)
 
 
 class VoiceRepublicIE(InfoExtractor):
-    _VALID_URL = r'https?://voicerepublic\.com/talks/(?P<id>[0-9a-z-]+)'
-    _TEST = {
-        'url': 'https://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
+    _VALID_URL = r'https?://voicerepublic\.com/(?:talks|embed)/(?P<id>[0-9a-z-]+)'
+    _TESTS = [{
+        'url': 'http://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
         'md5': '0554a24d1657915aa8e8f84e15dc9353',
         'info_dict': {
             'id': '2296',
+            'display_id': 'watching-the-watchers-building-a-sousveillance-state',
             'ext': 'm4a',
             'title': 'Watching the Watchers: Building a Sousveillance State',
-            'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
             'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
+            'duration': 1800,
+            'view_count': int,
         }
-    }
+    }, {
+        'url': 'http://voicerepublic.com/embed/watching-the-watchers-building-a-sousveillance-state',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        req = compat_urllib_request.Request(url)
+
+        req = compat_urllib_request.Request(
+            compat_urlparse.urljoin(url, '/talks/%s' % display_id))
         # Older versions of Firefox get redirected to an "upgrade browser" page
         req.add_header('User-Agent', 'youtube-dl')
         webpage = self._download_webpage(req, display_id)
-        thumbnail = self._og_search_thumbnail(webpage)
-        video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
 
-        if '<a>Queued for processing, please stand by...</a>' in webpage:
-            raise ExtractorError('Audio is still queued for processing')
+        if '>Queued for processing, please stand by...<' in webpage:
+            raise ExtractorError(
+                'Audio is still queued for processing', expected=True)
 
-        formats = [{
-            'url': 'https://voicerepublic.com' + path,
-            'ext': ext,
-            'format_id': ext,
-            'vcodec': 'none',
-        } for ext, path in re.findall(r"data-([^=]+)='(/[^']+\.\1)'", webpage)]
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)return ({.+?});\s*\n', webpage,
+                'data', default=None),
+            display_id, fatal=False)
+
+        if data:
+            title = data['title']
+            description = data.get('teaser')
+            talk_id = data.get('talk_id') or display_id
+            talk = data['talk']
+            duration = int_or_none(talk.get('duration'))
+            formats = [{
+                'url': compat_urlparse.urljoin(url, talk_url),
+                'format_id': format_id,
+                'ext': determine_ext(talk_url) or format_id,
+                'vcodec': 'none',
+            } for format_id, talk_url in talk['links'].items()]
+        else:
+            title = self._og_search_title(webpage)
+            description = self._html_search_regex(
+                r"(?s)<div class='talk-teaser'[^>]*>(.+?)</div>",
+                webpage, 'description', fatal=False)
+            talk_id = self._search_regex(
+                [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
+                webpage, 'talk id', default=None) or display_id
+            duration = None
+            formats = [{
+                'url': compat_urlparse.urljoin(url, talk_url),
+                'format_id': format_id,
+                'ext': determine_ext(talk_url) or format_id,
+                'vcodec': 'none',
+            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", webpage)]
         self._sort_formats(formats)
 
+        thumbnail = self._og_search_thumbnail(webpage)
+        view_count = int_or_none(self._search_regex(
+            r"class='play-count[^']*'>\s*(\d+) plays",
+            webpage, 'play count', fatal=False))
+
         return {
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-            'formats': formats,
-            'url': self._og_search_url(webpage),
+            'id': talk_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
             'thumbnail': thumbnail,
-            'description': self._og_search_description(webpage),
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
         }

From 370b39e8ece9f475d489eda721130eec9a9f15e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 18:37:52 +0600
Subject: [PATCH 0565/1515] [voicerepublic] Fix fallback branch formats
 extraction

---
 youtube_dl/extractor/voicerepublic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 1106c655b..254383d6c 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -47,12 +47,10 @@ class VoiceRepublicIE(InfoExtractor):
             raise ExtractorError(
                 'Audio is still queued for processing', expected=True)
 
-        data = self._parse_json(
-            self._search_regex(
-                r'(?s)return ({.+?});\s*\n', webpage,
-                'data', default=None),
-            display_id, fatal=False)
-
+        config = self._search_regex(
+            r'(?s)return ({.+?});\s*\n', webpage,
+            'data', default=None)
+        data = self._parse_json(config, display_id, fatal=False) if config else None
         if data:
             title = data['title']
             description = data.get('teaser')
@@ -74,12 +72,14 @@ class VoiceRepublicIE(InfoExtractor):
                 [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
                 webpage, 'talk id', default=None) or display_id
             duration = None
+            player = self._search_regex(
+                r"class='vr-player jp-jplayer'([^>]+)>", webpage, 'player')
             formats = [{
                 'url': compat_urlparse.urljoin(url, talk_url),
                 'format_id': format_id,
                 'ext': determine_ext(talk_url) or format_id,
                 'vcodec': 'none',
-            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", webpage)]
+            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", player)]
         self._sort_formats(formats)
 
         thumbnail = self._og_search_thumbnail(webpage)

From 95c5534f8ed016a81f715f291ab09c4ea2c3679c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 May 2015 17:41:11 +0200
Subject: [PATCH 0566/1515] ExecAfterDownloadPP, YoutubeDL: remove unused
 parameters

---
 youtube_dl/YoutubeDL.py                       | 1 -
 youtube_dl/__init__.py                        | 2 --
 youtube_dl/postprocessor/execafterdownload.py | 3 +--
 3 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d8583a8eb..4cf83c510 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -260,7 +260,6 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    exec_cmd:          Arbitrary command to run after downloading
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c88489f29..9cc9f851f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -246,7 +246,6 @@ def _real_main(argv=None):
     if opts.exec_cmd:
         postprocessors.append({
             'key': 'ExecAfterDownload',
-            'verboseOutput': opts.verbose,
             'exec_cmd': opts.exec_cmd,
         })
     if opts.xattr_set_filesize:
@@ -345,7 +344,6 @@ def _real_main(argv=None):
         'default_search': opts.default_search,
         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
         'encoding': opts.encoding,
-        'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 341437575..765fd8fe4 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -8,8 +8,7 @@ from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
-    def __init__(self, downloader=None, verboseOutput=None, exec_cmd=None):
-        self.verboseOutput = verboseOutput
+    def __init__(self, downloader=None, exec_cmd=None):
         self.exec_cmd = exec_cmd
 
     def run(self, information):

From 69b46b3d956220e4b3a3d5eda55768753a67f19d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 May 2015 17:47:49 +0200
Subject: [PATCH 0567/1515] ExecAfterDownloadPP: fix __init__ method

---
 youtube_dl/postprocessor/execafterdownload.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 765fd8fe4..13794b7ba 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -8,7 +8,8 @@ from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
-    def __init__(self, downloader=None, exec_cmd=None):
+    def __init__(self, downloader, exec_cmd):
+        super(ExecAfterDownloadPP, self).__init__(downloader)
         self.exec_cmd = exec_cmd
 
     def run(self, information):

From 70484b9f8ae629ccb87e8c0569f8f4bf2dfdb0ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 00:26:39 +0600
Subject: [PATCH 0568/1515] [postprocessor/ffmpeg] Extract `check_outdated`
 method

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 214de39f9..211faf69a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -36,7 +36,9 @@ class FFmpegPostProcessor(PostProcessor):
     def check_version(self):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
+        self.check_outdated()
 
+    def check_outdated(self):
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
@@ -44,6 +46,8 @@ class FFmpegPostProcessor(PostProcessor):
                 self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
+            return True
+        return False
 
     @staticmethod
     def get_versions(downloader=None):

From 7fcb605b82796e79a5f559624808ca9404df1154 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 00:27:29 +0600
Subject: [PATCH 0569/1515] [YoutubeDL] Fallback to `-f best` when merger is
 outdated

---
 youtube_dl/YoutubeDL.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cf83c510..7c3bdb964 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,9 +1086,10 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
-                    and info_dict['extractor'] in ['youtube', 'ted']
-                    and FFmpegMergerPP(self).available):
-                req_format_list.append('bestvideo+bestaudio')
+                    and info_dict['extractor'] in ['youtube', 'ted']):
+                merger = FFmpegMergerPP(self)
+                if merger.available and not merger.check_outdated():
+                    req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
         formats_to_download = []

From 13763ce599c8fbba43e57d2d79a9b007cfbd4ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 02:00:31 +0600
Subject: [PATCH 0570/1515] [postprocessor/ffmpeg] Add `can_merge` method

---
 youtube_dl/postprocessor/ffmpeg.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 211faf69a..cc65b34e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -36,9 +36,7 @@ class FFmpegPostProcessor(PostProcessor):
     def check_version(self):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
-        self.check_outdated()
 
-    def check_outdated(self):
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
@@ -46,8 +44,6 @@ class FFmpegPostProcessor(PostProcessor):
                 self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
-            return True
-        return False
 
     @staticmethod
     def get_versions(downloader=None):
@@ -595,6 +591,23 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return info['__files_to_merge'], info
 
+    def can_merge(self):
+        # TODO: figure out merge-capable ffmpeg version
+        if self.basename != 'avconv':
+            return True
+
+        required_version = '10-0'
+        if is_outdated_version(
+                self._versions[self.basename], required_version):
+            warning = ('Your copy of %s is outdated and unable to properly mux separate video and audio files, '
+                       'youtube-dl will download single file media. '
+                       'Update %s to version %s or newer to fix this.') % (
+                           self.basename, self.basename, required_version)
+            if self._downloader:
+                self._downloader.report_warning(warning)
+            return False
+        return True
+
 
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):

From 97fcf1bbd07ae0c5b6e530dcf2623d199452a76c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 02:01:16 +0600
Subject: [PATCH 0571/1515] [YoutubeDL] Check if merger can actually merge

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7c3bdb964..00f86b342 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1088,7 +1088,7 @@ class YoutubeDL(object):
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
                     and info_dict['extractor'] in ['youtube', 'ted']):
                 merger = FFmpegMergerPP(self)
-                if merger.available and not merger.check_outdated():
+                if merger.available and merger.can_merge():
                     req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From e41f450f2860ab5aa3f3a04bc646594c6dbc6714 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 May 2015 20:04:05 +0800
Subject: [PATCH 0572/1515] [tmz] Add support for articles (fixes #5477)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/tmz.py      | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e808f2734..b376fd279 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -543,7 +543,10 @@ from .thesixtyone import TheSixtyOneIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
-from .tmz import TMZIE
+from .tmz import (
+    TMZIE,
+    TMZArticleIE,
+)
 from .tnaflix import TNAFlixIE
 from .thvideo import (
     THVideoIE,
diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index c5c6fdc51..7dbe68b5c 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -30,3 +30,31 @@ class TMZIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'thumbnail': self._html_search_meta('ThumbURL', webpage),
         }
+
+
+class TMZArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
+    _TEST = {
+        'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
+        'md5': 'e482a414a38db73087450e3a6ce69d00',
+        'info_dict': {
+            'id': '0_6snoelag',
+            'ext': 'mp4',
+            'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
+            'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        embedded_video_info_str = self._html_search_regex(
+            r'tmzVideoEmbedV2\("([^)]+)"\);', webpage, 'embedded video info')
+
+        embedded_video_info = self._parse_json(
+            embedded_video_info_str, video_id,
+            transform_source=lambda s: s.replace('\\', ''))
+
+        return self.url_result(
+            'http://www.tmz.com/videos/%s/' % embedded_video_info['id'])

From 1f92865494c6efa1a0d5d90ffa849e85b80c8248 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 21:05:39 +0600
Subject: [PATCH 0573/1515] [dumpert] Add cpc cookie (Closes #5672)

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 9c594b757..999fb5620 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -26,7 +26,7 @@ class DumpertIE(InfoExtractor):
         video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'nsfw=1')
+        req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)
 
         files_base64 = self._search_regex(

From 511565282861e26913caddc1bcc0c865a9eec786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 21:31:36 +0600
Subject: [PATCH 0574/1515] [zingmp3] Capture error message

---
 youtube_dl/extractor/zingmp3.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 1afbe68ed..7dc1e2f2b 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -4,12 +4,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    @staticmethod
-    def _extract_item(item):
+    def _extract_item(self, item):
+        error_message = item.find('./errormessage').text
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message),
+                expected=True)
+
         title = item.find('./title').text.strip()
         source = item.find('./source').text
         extension = item.attrib['type']

From 81ed3bb9c0edb9a11b43964459ef57cca5683461 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Mon, 11 May 2015 17:38:08 +0200
Subject: [PATCH 0575/1515] [southpark] Sort alphabetically

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b376fd279..3368edf7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -489,8 +489,8 @@ from .soundgasm import (
 )
 from .southpark import (
     SouthParkIE,
-    SouthParkEsIE,
     SouthParkDeIE,
+    SouthParkEsIE,
     SouthParkNlIE
 )
 from .space import SpaceIE

From 968ee176777a1bf4e33cfb849a7241b0ac45d254 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Mon, 11 May 2015 18:02:25 +0200
Subject: [PATCH 0576/1515] [southparkdk] Add extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3368edf7c..de19dfd7a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .soundgasm import (
 from .southpark import (
     SouthParkIE,
     SouthParkDeIE,
+    SouthParkDkIE,
     SouthParkEsIE,
     SouthParkNlIE
 )
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 59e31198c..83e5a7659 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -57,3 +57,13 @@ class SouthParkNlIE(SouthParkIE):
         'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
         'playlist_count': 4,
     }]
+
+class SouthParkDkIE(SouthParkIE):
+    IE_NAME = 'southpark.dk'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'
+
+    _TESTS = [{
+        'url': 'http://www.southparkstudios.dk/full-episodes/s18e07-grounded-vindaloop',
+        'playlist_count': 4,
+    }]

From 6d3f5935e516760964052718e6b90324c6f07391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 23:47:50 +0600
Subject: [PATCH 0577/1515] [southpark] Fix IE_NAME

---
 youtube_dl/extractor/southpark.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 83e5a7659..7fb165a87 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -58,8 +58,9 @@ class SouthParkNlIE(SouthParkIE):
         'playlist_count': 4,
     }]
 
+
 class SouthParkDkIE(SouthParkIE):
-    IE_NAME = 'southpark.dk'
+    IE_NAME = 'southparkstudios.dk'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'
 

From d4b963d0a68f81f4fef5495af14e2e41add21a0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 May 2015 01:54:56 +0600
Subject: [PATCH 0578/1515] [vine] Relax `alt_title` (Closes #5677)

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 65c459fad..c733a48fa 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -75,7 +75,7 @@ class VineIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'alt_title': self._og_search_description(webpage),
+            'alt_title': self._og_search_description(webpage, default=None),
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From 5332fd91bf16867b6777bd6cfd0b5086f84112c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:42:13 +0800
Subject: [PATCH 0579/1515] [nytimes] Correct _VALID_URL of NYTimesArticleIE

---
 youtube_dl/extractor/nytimes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 6ffbe3863..7f254b867 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -89,7 +89,7 @@ class NYTimesIE(NYTimesBaseIE):
 
 
 class NYTimesArticleIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
     _TESTS = [{
         'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
         'md5': 'e2076d58b4da18e6a001d53fd56db3c9',

From 7dff03636a843a6990e52200edb3ecca1246b3df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:47:37 +0800
Subject: [PATCH 0580/1515] [utils] Support 'dur' field in TTML

---
 test/test_utils.py  | 2 +-
 youtube_dl/utils.py | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 86b110a7d..b40107037 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -600,7 +600,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                 <div xml:lang="en">
                     <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
                     <p begin="1" end="2">第二行<br/>♪♪</p>
-                    <p begin="2" end="3"><span>Third<br/>Line</span></p>
+                    <p begin="2" dur="1"><span>Third<br/>Line</span></p>
                 </div>
             </body>
             </tt>'''
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d73efcf25..5439fcb35 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1866,10 +1866,14 @@ def dfxp2srt(dfxp_data):
     paras = dfxp.findall(_x('.//ttml:p'))
 
     for para, index in zip(paras, itertools.count(1)):
+        begin_time = parse_dfxp_time_expr(para.attrib['begin'])
+        end_time = parse_dfxp_time_expr(para.attrib.get('end'))
+        if not end_time:
+            end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
-            format_srt_time(parse_dfxp_time_expr(para.attrib.get('begin'))),
-            format_srt_time(parse_dfxp_time_expr(para.attrib.get('end'))),
+            format_srt_time(begin_time),
+            format_srt_time(end_time),
             parse_node(para)))
 
     return ''.join(out)

From 1c7e2e64f6328024711d5fa999d4498396f4cb5c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:55:14 +0800
Subject: [PATCH 0581/1515] [nrk] Remove TTML to srt conversion codes

A common routine is implemented in utils.py and can be used via
--convert-subtitles.
---
 youtube_dl/extractor/nrk.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index e91d3a248..cc70c2950 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -200,20 +199,10 @@ class NRKTVIE(InfoExtractor):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
         captions = self._download_xml(
-            url, video_id, 'Downloading subtitles',
-            transform_source=lambda s: s.replace(r'<br />', '\r\n'))
+            url, video_id, 'Downloading subtitles')
         lang = captions.get('lang', 'no')
-        ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
-        srt = ''
-        for pos, p in enumerate(ps):
-            begin = parse_duration(p.get('begin'))
-            duration = parse_duration(p.get('dur'))
-            starttime = self._subtitles_timecode(begin)
-            endtime = self._subtitles_timecode(begin + duration)
-            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},
-            {'ext': 'srt', 'data': srt},
         ]}
 
     def _extract_f4m(self, manifest_url, video_id):

From c1c924abfeda45f29b991bb74f315f0e79dcf126 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 13:04:54 +0800
Subject: [PATCH 0582/1515] [utils,common] Merge format_srt_time and
 _subtitles_timecode

format_srt_time uses a comma as the delimiter between seconds and
milliseconds while _subtitles_timecode uses a dot. All .srt examples I
found on the Internet uses a comma, so I use a comma in the merged
version. See http://matroska.org/technical/specs/subtitles/srt.html and
http://devel.aegisub.org/wiki/SubtitleFormats/SRT
---
 youtube_dl/extractor/common.py    |  3 ---
 youtube_dl/extractor/kanalplay.py |  5 +++--
 youtube_dl/utils.py               | 12 ++++--------
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 981e34bc7..65bb77086 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1072,9 +1072,6 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
-    def _subtitles_timecode(self, seconds):
-        return '%02d:%02d:%02d.%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
-
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 2bb078036..4597d1b96 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
+    srt_subtitles_timecode,
 )
 
 
@@ -39,8 +40,8 @@ class KanalPlayIE(InfoExtractor):
             '%s\r\n%s --> %s\r\n%s'
             % (
                 num,
-                self._subtitles_timecode(item['startMillis'] / 1000.0),
-                self._subtitles_timecode(item['endMillis'] / 1000.0),
+                srt_subtitles_timecode(item['startMillis'] / 1000.0),
+                srt_subtitles_timecode(item['endMillis'] / 1000.0),
                 item['text'],
             ) for num, item in enumerate(subs, 1))
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5439fcb35..ed9ed9ed6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1835,12 +1835,8 @@ def parse_dfxp_time_expr(time_expr):
         return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
 
 
-def format_srt_time(seconds):
-    (mins, secs) = divmod(seconds, 60)
-    (hours, mins) = divmod(mins, 60)
-    millisecs = (secs - int(secs)) * 1000
-    secs = int(secs)
-    return '%02d:%02d:%02d,%03d' % (hours, mins, secs, millisecs)
+def srt_subtitles_timecode(seconds):
+    return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
 
 
 def dfxp2srt(dfxp_data):
@@ -1872,8 +1868,8 @@ def dfxp2srt(dfxp_data):
             end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
-            format_srt_time(begin_time),
-            format_srt_time(end_time),
+            srt_subtitles_timecode(begin_time),
+            srt_subtitles_timecode(end_time),
             parse_node(para)))
 
     return ''.join(out)

From 41333b97b9471316cf0f395db59196e6571fc776 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 12 May 2015 22:35:16 +0800
Subject: [PATCH 0583/1515] [qqmusic] Add support for charts / top lists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index de19dfd7a..8ec0c1032 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -414,6 +414,7 @@ from .qqmusic import (
     QQMusicIE,
     QQMusicSingerIE,
     QQMusicAlbumIE,
+    QQMusicToplistIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 174c8e0ae..d4a85d8c3 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    js_to_json,
 )
 from ..compat import compat_urllib_request
 
@@ -168,3 +169,57 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
             album_page, 'album details', default=None)
 
         return self.playlist_result(entries, mid, album_name, album_detail)
+
+
+class QQMusicToplistIE(QQPlaylistBaseIE):
+    _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
+    
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=toplist&p=global_12',
+        'info_dict': {
+            'id': 'global_12',
+            'title': 'itunes榜',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'http://y.qq.com/#type=toplist&p=top_6',
+        'info_dict': {
+            'id': 'top_6',
+            'title': 'QQ音乐巅峰榜·欧美',
+        },
+        'playlist_count': 100,
+    }]
+
+    @staticmethod
+    def strip_qq_jsonp(code):
+        return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
+    
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        list_type = list_id.split("_")[0]
+        num_id = list_id.split("_")[1]
+
+        list_page = self._download_webpage("http://y.qq.com/y/static/toplist/index/%s.html" % list_id, list_id, 'Download toplist page')
+        entries = []
+        if list_type == 'top':
+            list = self._download_json(
+                "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
+                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+
+            for song in list['l']:
+                s = song['s']
+                song_mid = s.split("|")[20]
+                entries.append(self.url_result(
+                    'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                    song_mid))
+
+        elif list_type == 'global':
+            entries = self.get_entries_from_page(list_page)
+
+        list_name = self._html_search_regex(
+            r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
+            default=None)
+        list_desc = None
+
+        return self.playlist_result(entries, list_id, list_name, list_desc)
\ No newline at end of file

From b480e7874b45862eae343ab8484aa43381cd28fa Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 12 May 2015 22:41:37 +0800
Subject: [PATCH 0584/1515] [qqmusic] Fix code formatting

---
 youtube_dl/extractor/qqmusic.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index d4a85d8c3..bca4a8f90 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -200,12 +200,15 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_type = list_id.split("_")[0]
         num_id = list_id.split("_")[1]
 
-        list_page = self._download_webpage("http://y.qq.com/y/static/toplist/index/%s.html" % list_id, list_id, 'Download toplist page')
+        list_page = self._download_webpage(
+            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
+            list_id, 'Download toplist page')
         entries = []
         if list_type == 'top':
             list = self._download_json(
                 "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
-                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', 
+                transform_source=self.strip_qq_jsonp)
 
             for song in list['l']:
                 s = song['s']
@@ -222,4 +225,5 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             default=None)
         list_desc = None
 
-        return self.playlist_result(entries, list_id, list_name, list_desc)
\ No newline at end of file
+        return self.playlist_result(entries, list_id, list_name, list_desc)
+        
\ No newline at end of file

From fd4eefed39595850b864d3be9711224e4e8e9dd4 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 13 May 2015 01:14:02 +0800
Subject: [PATCH 0585/1515] [qqmusic] Fix extraction for global list

---
 youtube_dl/extractor/qqmusic.py | 34 ++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bca4a8f90..3401dcaef 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -188,6 +188,13 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             'title': 'QQ音乐巅峰榜·欧美',
         },
         'playlist_count': 100,
+    }, {
+        'url': 'http://y.qq.com/#type=toplist&p=global_5',
+        'info_dict': {
+            'id': 'global_5',
+            'title': '韩国mnet排行榜',
+        },
+        'playlist_count': 50,
     }]
 
     @staticmethod
@@ -203,22 +210,23 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_page = self._download_webpage(
             "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
             list_id, 'Download toplist page')
+
         entries = []
+        jsonp_url = ""
         if list_type == 'top':
-            list = self._download_json(
-                "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
-                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', 
-                transform_source=self.strip_qq_jsonp)
-
-            for song in list['l']:
-                s = song['s']
-                song_mid = s.split("|")[20]
-                entries.append(self.url_result(
-                    'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
-                    song_mid))
-
+            jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
         elif list_type == 'global':
-            entries = self.get_entries_from_page(list_page)
+            jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
+        
+        list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 
+            errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+            
+        for song in list['l']:
+            s = song['s']
+            song_mid = s.split("|")[20]
+            entries.append(self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                song_mid))
 
         list_name = self._html_search_regex(
             r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',

From 86ec1e487c4908f4d0d0ece512007a2e5fedc593 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 13 May 2015 01:37:56 +0800
Subject: [PATCH 0586/1515] [qqmusic] Code fixes

---
 youtube_dl/extractor/qqmusic.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 3401dcaef..bae2ce31a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -212,10 +212,9 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             list_id, 'Download toplist page')
 
         entries = []
-        jsonp_url = ""
         if list_type == 'top':
             jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
-        elif list_type == 'global':
+        else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
         
         list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 

From 0b4253fa3710c656e12b6147ed7c1f7843bb9aae Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Tue, 12 May 2015 18:57:06 +0100
Subject: [PATCH 0587/1515] [BYUtvIE] Change thumbnail regex so test does not
 fail

---
 youtube_dl/extractor/byutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 6252be05b..3b2de517e 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -16,7 +16,7 @@ class BYUtvIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:5438d33774b6bdc662f9485a340401cc',
             'title': 'Season 5 Episode 5',
-            'thumbnail': 're:^https?://.*promo.*'
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
         'params': {
             'skip_download': True,

From 3749e36e9f0e6be2a3a3ab1b15c0c02be5a50e2f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 May 2015 21:16:45 +0800
Subject: [PATCH 0588/1515] [YoutubeDL] Fix PEP8 W503

---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cf83c510..84d50dab7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1085,9 +1085,9 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
-                    and info_dict['extractor'] in ['youtube', 'ted']
-                    and FFmpegMergerPP(self).available):
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
+                    info_dict['extractor'] in ['youtube', 'ted'] and
+                    FFmpegMergerPP(self).available):
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 372744c544ec3de1b35583e7d6fc2cbc4cc39f3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 May 2015 22:26:30 +0600
Subject: [PATCH 0589/1515] [odnoklassniki] Fix extraction (Closes #5671)

---
 youtube_dl/extractor/odnoklassniki.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 155d0ee6a..fbc521d1a 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -6,6 +6,7 @@ from ..utils import (
     unified_strdate,
     int_or_none,
     qualities,
+    unescapeHTML,
 )
 
 
@@ -36,8 +37,8 @@ class OdnoklassnikiIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         player = self._parse_json(
-            self._search_regex(
-                r"OKVideo\.start\(({.+?})\s*,\s*'VideoAutoplay_player'", webpage, 'player'),
+            unescapeHTML(self._search_regex(
+                r'data-attributes="([^"]+)"', webpage, 'player')),
             video_id)
 
         metadata = self._parse_json(player['flashvars']['metadata'], video_id)

From 8e595397529abc71093264e3695fb00d95be4d78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 02:32:00 +0800
Subject: [PATCH 0590/1515] [postprocessor/embedthumbnail] Use thumbnails
 downloaded by YoutubeDL

---
 youtube_dl/YoutubeDL.py                    |  2 +-
 youtube_dl/__init__.py                     |  8 +++++++-
 youtube_dl/postprocessor/embedthumbnail.py | 23 +++++++++++-----------
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 84d50dab7..0fbfe9642 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1848,7 +1848,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
-            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
 
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 9cc9f851f..ace17857c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -240,7 +240,13 @@ def _real_main(argv=None):
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
     if opts.embedthumbnail:
-        postprocessors.append({'key': 'EmbedThumbnail'})
+        already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
+        postprocessors.append({
+            'key': 'EmbedThumbnail',
+            'already_have_thumbnail': already_have_thumbnail
+        })
+        if not already_have_thumbnail:
+            opts.writethumbnail = True
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 4868a42fd..95c52f65f 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -7,11 +7,7 @@ import subprocess
 
 from .ffmpeg import FFmpegPostProcessor
 
-from ..compat import (
-    compat_urlretrieve,
-)
 from ..utils import (
-    determine_ext,
     check_executable,
     encodeFilename,
     PostProcessingError,
@@ -25,26 +21,30 @@ class EmbedThumbnailPPError(PostProcessingError):
 
 
 class EmbedThumbnailPP(FFmpegPostProcessor):
+    def __init__(self, downloader=None, already_have_thumbnail=False):
+        super(EmbedThumbnailPP, self).__init__(downloader)
+        self._already_have_thumbnail = already_have_thumbnail
+
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = filename + '.' + determine_ext(info['thumbnail'])
 
-        if not info.get('thumbnail'):
+        if not info.get('thumbnails'):
             raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')
 
-        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
+        thumbnail_filename = info['thumbnails'][-1]['filename']
 
         if info['ext'] == 'mp3':
             options = [
-                '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+                '-i', thumbnail_filename, '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
             self.run_ffmpeg(filename, temp_filename, options)
 
-            os.remove(encodeFilename(temp_thumbnail))
+            if not self._already_have_thumbnail:
+                os.remove(encodeFilename(thumbnail_filename))
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
@@ -52,7 +52,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
+            cmd = ['AtomicParsley', filename, '--artwork', thumbnail_filename, '-o', temp_filename]
 
             self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
 
@@ -66,7 +66,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 msg = stderr.decode('utf-8', 'replace').strip()
                 raise EmbedThumbnailPPError(msg)
 
-            os.remove(encodeFilename(temp_thumbnail))
+            if not self._already_have_thumbnail:
+                os.remove(encodeFilename(thumbnail_filename))
             # for formats that don't support thumbnails (like 3gp) AtomicParsley
             # won't create to the temporary file
             if b'No changes' in stdout:

From bb8ca1d112e95cd3fe48fff5af980a62a9db2572 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 02:35:28 +0800
Subject: [PATCH 0591/1515] [postprocessor/embedthumbnail] Use
 run_ffmpeg_multiple_files

---
 youtube_dl/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 95c52f65f..4e08c2709 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -36,12 +36,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
 
         if info['ext'] == 'mp3':
             options = [
-                '-i', thumbnail_filename, '-c', 'copy', '-map', '0', '-map', '1',
+                '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
-            self.run_ffmpeg(filename, temp_filename, options)
+            self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options)
 
             if not self._already_have_thumbnail:
                 os.remove(encodeFilename(thumbnail_filename))

From 2cc6d135479c5dbd6e715a1e767c5be163cd22ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 04:41:30 +0800
Subject: [PATCH 0592/1515] [postprocessor/embedthumbnail] Encode arguments in
 calling AtomicParsley

---
 youtube_dl/postprocessor/embedthumbnail.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 4e08c2709..8f825f785 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -9,6 +9,7 @@ from .ffmpeg import FFmpegPostProcessor
 
 from ..utils import (
     check_executable,
+    encodeArgument,
     encodeFilename,
     PostProcessingError,
     prepend_extension,
@@ -52,7 +53,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = ['AtomicParsley', filename, '--artwork', thumbnail_filename, '-o', temp_filename]
+            cmd = [encodeFilename('AtomicParsley', True),
+                   encodeFilename(filename, True),
+                   encodeArgument('--artwork'),
+                   encodeFilename(thumbnail_filename, True),
+                   encodeArgument('-o'),
+                   encodeFilename(temp_filename, True)]
 
             self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
 

From 86c7fdb17c0dcbff88a8daa131fddc57b6304b83 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 14:26:47 +0800
Subject: [PATCH 0593/1515] [xattr] Enhance error handling to catch ENOSPC

Fixes #5589
---
 youtube_dl/postprocessor/xattrpp.py | 61 +++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 93d0abcf6..16f2966e9 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -3,18 +3,32 @@ from __future__ import unicode_literals
 import os
 import subprocess
 import sys
+import errno
 
 from .common import PostProcessor
-from ..compat import (
-    subprocess_check_output
-)
 from ..utils import (
     check_executable,
     hyphenate_date,
     version_tuple,
+    PostProcessingError,
+    encodeArgument,
+    encodeFilename,
 )
 
 
+class XAttrMetadataError(PostProcessingError):
+    def __init__(self, code=None, msg='Unknown error'):
+        super(XAttrMetadataError, self).__init__(msg)
+        self.code = code
+
+        # Parsing code and msg
+        if (self.code in (errno.ENOSPC, errno.EDQUOT) or
+                'No space left' in self.msg or 'Disk quota excedded' in self.msg):
+            self.reason = 'NO_SPACE'
+        else:
+            self.reason = 'NOT_SUPPORTED'
+
+
 class XAttrMetadataPP(PostProcessor):
 
     #
@@ -51,7 +65,10 @@ class XAttrMetadataPP(PostProcessor):
                 raise ImportError
 
             def write_xattr(path, key, value):
-                return xattr.setxattr(path, key, value)
+                try:
+                    xattr.set(path, key, value)
+                except EnvironmentError as e:
+                    raise XAttrMetadataError(e.errno, e.strerror)
 
         except ImportError:
             if os.name == 'nt':
@@ -62,8 +79,11 @@ class XAttrMetadataPP(PostProcessor):
                     assert os.path.exists(path)
 
                     ads_fn = path + ":" + key
-                    with open(ads_fn, "wb") as f:
-                        f.write(value)
+                    try:
+                        with open(ads_fn, "wb") as f:
+                            f.write(value)
+                    except EnvironmentError as e:
+                        raise XAttrMetadataError(e.errno, e.strerror)
             else:
                 user_has_setfattr = check_executable("setfattr", ['--version'])
                 user_has_xattr = check_executable("xattr", ['-h'])
@@ -71,12 +91,24 @@ class XAttrMetadataPP(PostProcessor):
                 if user_has_setfattr or user_has_xattr:
 
                     def write_xattr(path, key, value):
+                        value = value.decode('utf-8')
                         if user_has_setfattr:
-                            cmd = ['setfattr', '-n', key, '-v', value, path]
+                            executable = 'setfattr'
+                            opts = ['-n', key, '-v', value]
                         elif user_has_xattr:
-                            cmd = ['xattr', '-w', key, value, path]
+                            executable = 'xattr'
+                            opts = ['-w', key, value]
 
-                        subprocess_check_output(cmd)
+                        cmd = ([encodeFilename(executable, True)] +
+                               [encodeArgument(o) for o in opts] +
+                               [encodeFilename(path, True)])
+
+                        p = subprocess.Popen(
+                            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        stdout, stderr = p.communicate()
+                        stderr = stderr.decode('utf-8', 'replace')
+                        if p.returncode != 0:
+                            raise XAttrMetadataError(p.returncode, stderr)
 
                 else:
                     # On Unix, and can't find pyxattr, setfattr, or xattr.
@@ -121,6 +153,13 @@ class XAttrMetadataPP(PostProcessor):
 
             return [], info
 
-        except (subprocess.CalledProcessError, OSError):
-            self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
+        except XAttrMetadataError as e:
+            if e.reason == 'NO_SPACE':
+                self._downloader.report_warning(
+                    'There\'s no disk space left or disk quota exceeded. ' +
+                    'Extended attributes are not written.')
+            else:
+                self._downloader.report_error(
+                    'This filesystem doesn\'t support extended attributes. ' +
+                    '(You may have to enable them in your /etc/fstab)')
             return [], info

From fbff30d2dbc6462c628384ea5960c2461e7cdcca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 14:51:00 +0800
Subject: [PATCH 0594/1515] [xattr] Catch 'Argument list too long'

---
 youtube_dl/postprocessor/xattrpp.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 16f2966e9..27e273000 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -25,6 +25,8 @@ class XAttrMetadataError(PostProcessingError):
         if (self.code in (errno.ENOSPC, errno.EDQUOT) or
                 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
             self.reason = 'NO_SPACE'
+        elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
+            self.reason = 'VALUE_TOO_LONG'
         else:
             self.reason = 'NOT_SUPPORTED'
 
@@ -103,8 +105,11 @@ class XAttrMetadataPP(PostProcessor):
                                [encodeArgument(o) for o in opts] +
                                [encodeFilename(path, True)])
 
-                        p = subprocess.Popen(
-                            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        try:
+                            p = subprocess.Popen(
+                                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        except EnvironmentError as e:
+                            raise XAttrMetadataError(e.errno, e.strerror)
                         stdout, stderr = p.communicate()
                         stderr = stderr.decode('utf-8', 'replace')
                         if p.returncode != 0:
@@ -158,6 +163,9 @@ class XAttrMetadataPP(PostProcessor):
                 self._downloader.report_warning(
                     'There\'s no disk space left or disk quota exceeded. ' +
                     'Extended attributes are not written.')
+            elif e.reason == 'VALUE_TOO_LONG':
+                self._downloader.report_warning(
+                    'Unable to write extended attributes due to too long values.')
             else:
                 self._downloader.report_error(
                     'This filesystem doesn\'t support extended attributes. ' +

From 509c630db8cdaff473f95805cda1ae350107e36b Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 14 May 2015 08:09:56 +0100
Subject: [PATCH 0595/1515] [CanalplusIE] Update tests that were no longer
 working

---
 youtube_dl/extractor/canalplus.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 1b14471e5..71801488a 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -25,14 +25,13 @@ class CanalplusIE(InfoExtractor):
     }
 
     _TESTS = [{
-        'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
-        'md5': '3db39fb48b9685438ecf33a1078023e4',
+        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
         'info_dict': {
-            'id': '922470',
+            'id': '1263092',
             'ext': 'flv',
-            'title': 'Zapping - 26/08/13',
-            'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
-            'upload_date': '20130826',
+            'title': 'Le Zapping - 13/05/15',
+            'description': 'md5:09738c0d06be4b5d06a0940edb0da73f',
+            'upload_date': '20150513',
         },
     }, {
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
@@ -56,7 +55,6 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
-        'md5': '65aa83ad62fe107ce29e564bb8712580',
         'info_dict': {
             'id': '1213714',
             'ext': 'flv',

From c827d4cfdb9ce47d13ccbec32d2b32dfb429ea8a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 16:53:10 +0800
Subject: [PATCH 0596/1515] [xattr] Enhanced error messages on Windows

---
 youtube_dl/postprocessor/xattrpp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 27e273000..7d88e1308 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -167,7 +167,10 @@ class XAttrMetadataPP(PostProcessor):
                 self._downloader.report_warning(
                     'Unable to write extended attributes due to too long values.')
             else:
-                self._downloader.report_error(
-                    'This filesystem doesn\'t support extended attributes. ' +
-                    '(You may have to enable them in your /etc/fstab)')
+                msg = 'This filesystem doesn\'t support extended attributes. '
+                if os.name == 'nt':
+                    msg += 'You need to use NTFS.'
+                else:
+                    msg += '(You may have to enable them in your /etc/fstab)'
+                self._downloader.report_error(msg)
             return [], info

From 7d57d2e18be416faa593364966ccf667243fd3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 14:59:27 +0600
Subject: [PATCH 0597/1515] [canalplus] Restore checksums in tests

---
 youtube_dl/extractor/canalplus.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 71801488a..699b4f7d0 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -26,6 +26,7 @@ class CanalplusIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
+        'md5': 'b3481d7ca972f61e37420798d0a9d934',
         'info_dict': {
             'id': '1263092',
             'ext': 'flv',
@@ -55,6 +56,7 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
+        'md5': 'f3a46edcdf28006598ffaf5b30e6a2d4',
         'info_dict': {
             'id': '1213714',
             'ext': 'flv',

From 82245a6de77f4755b063310258c5611c15f5ffbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 15:21:27 +0600
Subject: [PATCH 0598/1515] [YoutubeDL] Restore filename for thumbnails

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c8196d08..691f3e09f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1849,7 +1849,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
-            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
 
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %

From fa6a16996e4a1aeee4e421b172efc6c351b1b123 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 18:00:57 +0800
Subject: [PATCH 0599/1515] [worldstarhiphop] Support Android URLs (fixes
 #5629)

---
 youtube_dl/extractor/worldstarhiphop.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index d5c26a032..a3ea26feb 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 
 
 class WorldStarHipHopIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/videos/video\.php\?v=(?P<id>.*)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
+    _TESTS = [{
         "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
         "md5": "9d04de741161603bf7071bbf4e883186",
         "info_dict": {
@@ -15,7 +15,15 @@ class WorldStarHipHopIE(InfoExtractor):
             "ext": "mp4",
             "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
         }
-    }
+    }, {
+        'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
+        'md5': 'dc1c76c83ecc4190bb1eb143899b87d3',
+        'info_dict': {
+            'id': 'wshh6a7q1ny0G34ZwuIO',
+            'ext': 'mp4',
+            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -26,19 +34,22 @@ class WorldStarHipHopIE(InfoExtractor):
             return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')
 
         video_url = self._search_regex(
-            r'so\.addVariable\("file","(.*?)"\)', webpage, 'video URL')
+            [r'so\.addVariable\("file","(.*?)"\)',
+             r'<div class="artlist">\s*<a[^>]+href="([^"]+)">'],
+            webpage, 'video URL')
 
         if 'youtube' in video_url:
             return self.url_result(video_url, ie='Youtube')
 
         video_title = self._html_search_regex(
-            r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+            [r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+             r'<span[^>]+class="tc-sp-pinned-title">(.*)</span>'],
             webpage, 'title')
 
         # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
         thumbnail = self._html_search_regex(
             r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
-            fatal=False)
+            default=None)
         if not thumbnail:
             _title = r'candytitles.*>(.*)</span>'
             mobj = re.search(_title, webpage)

From 7a012d5a16632a103466f9e9794dd98ad573ce88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 16:39:35 +0600
Subject: [PATCH 0600/1515] [screenwavemedia] Add support for player2 URLs
 (Closes #5696)

---
 youtube_dl/extractor/screenwavemedia.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 74fb1983a..d1ab66b32 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
+    _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
@@ -20,7 +20,10 @@ class ScreenwaveMediaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
+
+        playerdata = self._download_webpage(
+            'http://player.screenwavemedia.com/play/player.php?id=%s' % video_id,
+            video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
             r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
@@ -99,7 +102,7 @@ class TeamFourIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            r'src="(http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
             webpage, 'player data URL')
 
         video_title = self._html_search_regex(

From 548897396158d7822020f45c10301e9ca3c46453 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:25:43 +0800
Subject: [PATCH 0601/1515] [qqmusic] flake8

---
 youtube_dl/extractor/qqmusic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bae2ce31a..5ddbb183e 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -173,7 +173,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
-    
+
     _TESTS = [{
         'url': 'http://y.qq.com/#type=toplist&p=global_12',
         'info_dict': {
@@ -200,7 +200,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     @staticmethod
     def strip_qq_jsonp(code):
         return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
-    
+
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
@@ -208,7 +208,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         num_id = list_id.split("_")[1]
 
         list_page = self._download_webpage(
-            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
+            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
             list_id, 'Download toplist page')
 
         entries = []
@@ -216,10 +216,11 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
         else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
-        
-        list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 
+
+        list = self._download_json(
+            jsonp_url, list_id, note='Retrieve toplist json',
             errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
-            
+
         for song in list['l']:
             s = song['s']
             song_mid = s.split("|")[20]
@@ -233,4 +234,3 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_desc = None
 
         return self.playlist_result(entries, list_id, list_name, list_desc)
-        
\ No newline at end of file

From 29ea57283e473e94c72cf9cbc065c3c05a14830f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:28:42 +0800
Subject: [PATCH 0602/1515] [qqmusic] Refactoring QQMusicToplistIE

---
 youtube_dl/extractor/qqmusic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 5ddbb183e..59e93a1dd 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -204,8 +204,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        list_type = list_id.split("_")[0]
-        num_id = list_id.split("_")[1]
+        list_type, num_id = list_id.split("_")
 
         list_page = self._download_webpage(
             "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
@@ -217,11 +216,11 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
 
-        list = self._download_json(
+        toplist_json = self._download_json(
             jsonp_url, list_id, note='Retrieve toplist json',
             errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
 
-        for song in list['l']:
+        for song in toplist_json['l']:
             s = song['s']
             song_mid = s.split("|")[20]
             entries.append(self.url_result(
@@ -231,6 +230,5 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_name = self._html_search_regex(
             r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
             default=None)
-        list_desc = None
 
-        return self.playlist_result(entries, list_id, list_name, list_desc)
+        return self.playlist_result(entries, list_id, list_name)

From 7ec676bb3dd6cba4b56fccb2d5aae08e66086b4e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:32:36 +0800
Subject: [PATCH 0603/1515] [qqmusic] Add IE_NAME for all extractors

---
 youtube_dl/extractor/qqmusic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 59e93a1dd..13113820b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -15,6 +15,7 @@ from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
+    IE_NAME = 'qqmusic'
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
@@ -97,6 +98,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 
 
 class QQMusicSingerIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:singer'
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -140,6 +142,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
 
 class QQMusicAlbumIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:album'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TEST = {
@@ -172,6 +175,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:toplist'
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{

From 1ae72fb23df709687091133602fd715ab6cb7b2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 22:28:42 +0600
Subject: [PATCH 0604/1515] [soundcloud:user] Defer download link resolve
 (Closes #5248)

Looks like final download links can expire before downloading process reach them. So, resolving download links right before actual downloading.
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 183ff50f4..c23c5ee0f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -336,7 +336,7 @@ class SoundcloudUserIE(SoundcloudIE):
             if len(new_entries) == 0:
                 self.to_screen('%s: End page received' % uploader)
                 break
-            entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
+            entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries)
 
         return {
             '_type': 'playlist',

From 3a105f7b20e8a3f742ac86cc1a6b02935b831778 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 May 2015 02:17:22 +0800
Subject: [PATCH 0605/1515] [teamcoco] Rewrite preload data extraction

Idea: "puncture" some consecutive fragments and check whether the
b64decode result of a punctured string is a valid JSON or not.

It's a O(N^3) algorithm, but should be fast for a small N (less than 30
fragments in all test cases)
---
 youtube_dl/extractor/teamcoco.py | 53 ++++++++++++++++----------------
 1 file changed, 26 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 95d58ddd0..410eb7d3a 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import base64
 import binascii
 import re
+import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -68,41 +69,39 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        data = preload = None
-        preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
-        if preloads:
-            preload = max([(len(p), p) for p in preloads])[1]
+        data = None
 
-        if not preload:
-            preload = ''.join(re.findall(r'this\.push\("([^"]+)"\);', webpage))
+        preload_codes = self._html_search_regex(
+            r'(function.+)setTimeout\(function\(\)\{playlist',
+            webpage, 'preload codes')
+        base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+        base64_fragments.remove('init')
 
-        if not preload:
-            preload = self._html_search_regex([
-                r'player,\[?"([^"]+)"\]?', r'player.init\(\[?"([^"]+)"\]?\)'
-            ], webpage.replace('","', ''), 'preload data', default=None)
-
-        if not preload:
-            preload_codes = self._html_search_regex(
-                r'(function.+)setTimeout\(function\(\)\{playlist',
-                webpage, 'preload codes')
-            base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
-            base64_fragments.remove('init')
-            for i in range(len(base64_fragments)):
-                cur_sequence = (''.join(base64_fragments[i:] + base64_fragments[:i])).encode('ascii')
+        def _check_sequence(cur_fragments):
+            if not cur_fragments:
+                return
+            for i in range(len(cur_fragments)):
+                cur_sequence = (''.join(cur_fragments[i:] + cur_fragments[:i])).encode('ascii')
                 try:
                     raw_data = base64.b64decode(cur_sequence)
-                except (TypeError, binascii.Error):
+                    if compat_ord(raw_data[0]) == compat_ord('{'):
+                        return json.loads(raw_data.decode('utf-8'))
+                except (TypeError, binascii.Error, UnicodeDecodeError, ValueError):
                     continue
-                if compat_ord(raw_data[0]) == compat_ord('{'):
-                    data = self._parse_json(raw_data.decode('utf-8'), video_id, fatal=False)
 
-        if not preload and not data:
-            raise ExtractorError(
-                'Preload information could not be extracted', expected=True)
+        def _check_data():
+            for i in range(len(base64_fragments) + 1):
+                for j in range(i, len(base64_fragments) + 1):
+                    data = _check_sequence(base64_fragments[:i] + base64_fragments[j:])
+                    if data:
+                        return data
+
+        self.to_screen('Try to compute possible data sequence. This may take some time.')
+        data = _check_data()
 
         if not data:
-            data = self._parse_json(
-                base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
+            raise ExtractorError(
+                'Preload information could not be extracted', expected=True)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From 12675275a1d2158fbe409361888569e4cb52ef07 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 May 2015 02:27:41 +0800
Subject: [PATCH 0606/1515] [teamcoco] Detect expired videos (#5626)

---
 youtube_dl/extractor/teamcoco.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 410eb7d3a..56be52638 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -62,7 +62,9 @@ class TeamcocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id)
+        webpage, urlh = self._download_webpage_handle(url, display_id)
+        if 'src=expired' in urlh.geturl():
+            raise ExtractorError('This video is expired.', expected=True)
 
         video_id = mobj.group('video_id')
         if not video_id:

From 2bc43303031215436b201e656094b60ab3ec7e9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 May 2015 23:41:27 +0200
Subject: [PATCH 0607/1515] [youtube:history] Fix extraction (fixes #5702)

It uses the same method as YoutubeSubscriptionsIE, if other feed starts using it we should consider using base class.
---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0869c9fd4..e58184adc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1667,13 +1667,42 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
         return self._extract_playlist('WL')
 
 
-class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+class YoutubeHistoryIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:history'
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
-    _FEED_NAME = 'history'
-    _PERSONAL_FEED = True
-    _PLAYLIST_TITLE = 'Youtube Watch History'
+    _TESTS = []
+
+    def _real_extract(self, url):
+        title = 'Youtube History'
+        page = self._download_webpage('https://www.youtube.com/feed/history', title)
+
+        # The extraction process is the same as for playlists, but the regex
+        # for the video ids doesn't contain an index
+        ids = []
+        more_widget_html = content_html = page
+
+        for page_num in itertools.count(1):
+            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
+            new_ids = orderedSet(matches)
+            ids.extend(new_ids)
+
+            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+            if not mobj:
+                break
+
+            more = self._download_json(
+                'https://youtube.com/%s' % mobj.group('more'), title,
+                'Downloading page #%s' % page_num,
+                transform_source=uppercase_escape)
+            content_html = more['content_html']
+            more_widget_html = more['load_more_widget_html']
+
+        return {
+            '_type': 'playlist',
+            'title': title,
+            'entries': self._ids_to_results(ids),
+        }
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):

From c4fc559f45ea5c40409eab44867ff2b4f08976c2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 May 2015 10:13:43 +0200
Subject: [PATCH 0608/1515] release 2015.05.15

---
 docs/supportedsites.md | 11 ++++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 98b625380..43fbe8b1d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -291,6 +291,7 @@
  - **MySpass**
  - **myvideo**
  - **MyVidster**
+ - **N-JOY**
  - **n-tv.de**
  - **NationalGeographic**
  - **Naver**
@@ -368,9 +369,10 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
- - **QQMusic**
- - **QQMusicAlbum**
- - **QQMusicSinger**
+ - **qqmusic**
+ - **qqmusic:album**
+ - **qqmusic:singer**
+ - **qqmusic:toplist**
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -439,6 +441,7 @@
  - **southpark.cc.com:español**
  - **southpark.de**
  - **southpark.nl**
+ - **southparkstudios.dk**
  - **Space**
  - **SpankBang**
  - **Spankwire**
@@ -492,6 +495,7 @@
  - **tlc.com**
  - **tlc.de**
  - **TMZ**
+ - **TMZArticle**
  - **TNAFlix**
  - **tou.tv**
  - **Toypics**: Toypics user profile
@@ -569,6 +573,7 @@
  - **vk.com**
  - **vk.com:user-videos**: vk.com:All of a user's videos
  - **Vodlocker**
+ - **VoiceRepublic**
  - **Vporn**
  - **VRT**
  - **vube**: Vube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 83c5a1659..38f00bc9b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.10'
+__version__ = '2015.05.15'

From 3884dcf313223040049e4153e0c398fbc36b5117 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 14:03:00 +0200
Subject: [PATCH 0609/1515] YoutubeDL: ignore indexes from 'playlist_items'
 that are not in the list (fixes #5706)

We ignore them instead of failing to match the behaviour of the 'playliststart' parameter.
---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 691f3e09f..5df889945 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -759,7 +759,9 @@ class YoutubeDL(object):
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
                 if playlistitems:
-                    entries = [ie_entries[i - 1] for i in playlistitems]
+                    entries = [
+                        ie_entries[i - 1] for i in playlistitems
+                        if -n_all_entries <= i - 1 < n_all_entries]
                 else:
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)

From e9eaf3fbcf497e76a55d2ba15d5880af83a065d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 14:06:19 +0200
Subject: [PATCH 0610/1515] [test/YoutubeDL] Add tests for 'playliststart',
 'playlistend' and 'playlist_items'

---
 test/test_YoutubeDL.py | 46 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 82b827536..a13c09ef4 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -12,6 +12,7 @@ import copy
 
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_str
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import match_filter_func
@@ -507,6 +508,51 @@ class TestYoutubeDL(unittest.TestCase):
         res = get_videos(f)
         self.assertEqual(res, ['1'])
 
+    def test_playlist_items_selection(self):
+        entries = [{
+            'id': compat_str(i),
+            'title': compat_str(i),
+            'url': TEST_URL,
+        } for i in range(1, 5)]
+        playlist = {
+            '_type': 'playlist',
+            'id': 'test',
+            'entries': entries,
+            'extractor': 'test:playlist',
+            'extractor_key': 'test:playlist',
+            'webpage_url': 'http://example.com',
+        }
+
+        def get_ids(params):
+            ydl = YDL(params)
+            # make a copy because the dictionary can be modified
+            ydl.process_ie_result(playlist.copy())
+            return [int(v['id']) for v in ydl.downloaded_info_dicts]
+
+        result = get_ids({})
+        self.assertEqual(result, [1, 2, 3, 4])
+
+        result = get_ids({'playlistend': 10})
+        self.assertEqual(result, [1, 2, 3, 4])
+
+        result = get_ids({'playlistend': 2})
+        self.assertEqual(result, [1, 2])
+
+        result = get_ids({'playliststart': 10})
+        self.assertEqual(result, [])
+
+        result = get_ids({'playliststart': 2})
+        self.assertEqual(result, [2, 3, 4])
+
+        result = get_ids({'playlist_items': '2-4'})
+        self.assertEqual(result, [2, 3, 4])
+
+        result = get_ids({'playlist_items': '2,4'})
+        self.assertEqual(result, [2, 4])
+
+        result = get_ids({'playlist_items': '10'})
+        self.assertEqual(result, [])
+
 
 if __name__ == '__main__':
     unittest.main()

From 15da7ce7fb89203247f4c959a748281ecf353e2a Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 15 May 2015 12:28:10 +0100
Subject: [PATCH 0611/1515] Fix file format extraction regex and update test
 file checksum

---
 youtube_dl/extractor/ccc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 2a5d4be18..6924eac70 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -16,7 +16,7 @@ class CCCIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
-        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'md5': '3a1eda8f3a29515d27f5adb967d7e740',
         'info_dict': {
             'id': '20131228183',
             'ext': 'mp4',
@@ -51,7 +51,7 @@ class CCCIE(InfoExtractor):
 
         matches = re.finditer(r'''(?xs)
             <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
-            <a\s+href='(?P<http_url>[^']+)'>\s*
+            <a\s+download\s+href='(?P<http_url>[^']+)'>\s*
             (?:
                 .*?
                 <a\s+href='(?P<torrent_url>[^']+\.torrent)'

From a7b8467ac0baecd02a815b1f57731ae9bb10ab87 Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 16:52:11 +0400
Subject: [PATCH 0612/1515] Sportbox extractor fix.

---
 youtube_dl/extractor/sportbox.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index becdf658f..830220543 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SportBoxIE(InfoExtractor):
-    _VALID_URL = r'https?://news\.sportbox\.ru/Vidy_sporta/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
+    _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
     _TESTS = [
         {
             'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
@@ -50,7 +50,7 @@ class SportBoxIE(InfoExtractor):
             display_id, 'Downloading player webpage')
 
         hls = self._search_regex(
-            r"var\s+original_hls_file\s*=\s*'([^']+)'", player, 'hls file')
+            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]+([^\"]+)['\"]+", player, 'hls file')
 
         formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
 

From ae670a6ed8019f1b69bbe345621f51c8b32789ec Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 17:53:05 +0400
Subject: [PATCH 0613/1515] Sportbox source fix. HD videos support.

---
 youtube_dl/extractor/sportbox.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 830220543..695b3ff82 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -14,7 +14,7 @@ class SportBoxIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
     _TESTS = [
         {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',            
             'md5': 'ff56a598c2cf411a9a38a69709e97079',
             'info_dict': {
                 'id': '80822',
@@ -42,11 +42,15 @@ class SportBoxIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._search_regex(
-            r'src="/vdl/player/media/(\d+)"', webpage, 'video id')
+        sobj = re.search(r'src="/vdl/player/(?P<media_type>\w+)/(?P<video_id>\d+)"', webpage)
+        if (sobj):
+            video_id = sobj.group('video_id')
+            media_type = sobj.group('media_type')
+        else:
+            raise RegexNotFoundError('Unable to extract video_id')
 
         player = self._download_webpage(
-            'http://news.sportbox.ru/vdl/player/media/%s' % video_id,
+            'http://news.sportbox.ru/vdl/player/%s/%s' % (media_type, video_id),
             display_id, 'Downloading player webpage')
 
         hls = self._search_regex(

From 25f14e9f93295a787e0cb436a5f6179d6174733d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 21:06:59 +0600
Subject: [PATCH 0614/1515] [youtube] Separate feed extractor

---
 youtube_dl/extractor/youtube.py | 143 +++++++++-----------------------
 1 file changed, 37 insertions(+), 106 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e58184adc..9096a2975 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -49,6 +49,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             # YouTube sets the expire time to about two months
             expire_time=time.time() + 2 * 30 * 24 * 3600)
 
+    def _ids_to_results(self, ids):
+        return [
+            self.url_result(vid_id, 'Youtube', video_id=vid_id)
+            for vid_id in ids]
+
     def _login(self):
         """
         Attempt to log in to YouTube.
@@ -1261,11 +1266,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _ids_to_results(self, ids):
-        return [
-            self.url_result(vid_id, 'Youtube', video_id=vid_id)
-            for vid_id in ids]
-
     def _extract_mix(self, playlist_id):
         # The mixes are generated from a single video
         # the id of the playlist is just 'RD' + video_id
@@ -1601,20 +1601,10 @@ class YoutubeShowIE(InfoExtractor):
 
 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
     """
-    Base class for extractors that fetch info from
-    http://www.youtube.com/feed_ajax
+    Base class for feed extractors
     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
     """
     _LOGIN_REQUIRED = True
-    # use action_load_personal_feed instead of action_load_system_feed
-    _PERSONAL_FEED = False
-
-    @property
-    def _FEED_TEMPLATE(self):
-        action = 'action_load_system_feed'
-        if self._PERSONAL_FEED:
-            action = 'action_load_personal_feed'
-        return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
 
     @property
     def IE_NAME(self):
@@ -1624,58 +1614,8 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        feed_entries = []
-        paging = 0
-        for i in itertools.count(1):
-            info = self._download_json(
-                self._FEED_TEMPLATE % paging,
-                '%s feed' % self._FEED_NAME,
-                'Downloading page %s' % i,
-                transform_source=uppercase_escape)
-            feed_html = info.get('feed_html') or info.get('content_html')
-            load_more_widget_html = info.get('load_more_widget_html') or feed_html
-            m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
-            ids = orderedSet(m.group(1) for m in m_ids)
-            feed_entries.extend(
-                self.url_result(video_id, 'Youtube', video_id=video_id)
-                for video_id in ids)
-            mobj = re.search(
-                r'data-uix-load-more-href="/?[^"]+paging=(?P<paging>\d+)',
-                load_more_widget_html)
-            if mobj is None:
-                break
-            paging = mobj.group('paging')
-        return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
-
-
-class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
-    IE_NAME = 'youtube:recommended'
-    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
-    _FEED_NAME = 'recommended'
-    _PLAYLIST_TITLE = 'Youtube Recommended videos'
-
-
-class YoutubeWatchLaterIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:watchlater'
-    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
-
-    _TESTS = []  # override PlaylistIE tests
-
-    def _real_extract(self, url):
-        return self._extract_playlist('WL')
-
-
-class YoutubeHistoryIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:history'
-    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
-    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
-    _TESTS = []
-
-    def _real_extract(self, url):
-        title = 'Youtube History'
-        page = self._download_webpage('https://www.youtube.com/feed/history', title)
+        page = self._download_webpage(
+            'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
 
         # The extraction process is the same as for playlists, but the regex
         # for the video ids doesn't contain an index
@@ -1692,17 +1632,25 @@ class YoutubeHistoryIE(YoutubePlaylistIE):
                 break
 
             more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), title,
+                'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
                 'Downloading page #%s' % page_num,
                 transform_source=uppercase_escape)
             content_html = more['content_html']
             more_widget_html = more['load_more_widget_html']
 
-        return {
-            '_type': 'playlist',
-            'title': title,
-            'entries': self._ids_to_results(ids),
-        }
+        return self.playlist_result(
+            self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
+
+
+class YoutubeWatchLaterIE(YoutubePlaylistIE):
+    IE_NAME = 'youtube:watchlater'
+    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
+
+    _TESTS = []  # override PlaylistIE tests
+
+    def _real_extract(self, url):
+        return self._extract_playlist('WL')
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
@@ -1717,42 +1665,25 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
         return self.url_result(playlist_id, 'YoutubePlaylist')
 
 
-class YoutubeSubscriptionsIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:subscriptions'
+class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
+    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
+    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
+    _FEED_NAME = 'recommended'
+    _PLAYLIST_TITLE = 'Youtube Recommended videos'
+
+
+class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
-    _TESTS = []
+    _FEED_NAME = 'subscriptions'
+    _PLAYLIST_TITLE = 'Youtube Subscriptions'
 
-    def _real_extract(self, url):
-        title = 'Youtube Subscriptions'
-        page = self._download_webpage('https://www.youtube.com/feed/subscriptions', title)
 
-        # The extraction process is the same as for playlists, but the regex
-        # for the video ids doesn't contain an index
-        ids = []
-        more_widget_html = content_html = page
-
-        for page_num in itertools.count(1):
-            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
-            new_ids = orderedSet(matches)
-            ids.extend(new_ids)
-
-            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-            if not mobj:
-                break
-
-            more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), title,
-                'Downloading page #%s' % page_num,
-                transform_source=uppercase_escape)
-            content_html = more['content_html']
-            more_widget_html = more['load_more_widget_html']
-
-        return {
-            '_type': 'playlist',
-            'title': title,
-            'entries': self._ids_to_results(ids),
-        }
+class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
+    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
+    _FEED_NAME = 'history'
+    _PLAYLIST_TITLE = 'Youtube History'
 
 
 class YoutubeTruncatedURLIE(InfoExtractor):

From 62c95fd5fcb8dbea2faeb4edac4c5177cbac5912 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 21:42:34 +0600
Subject: [PATCH 0615/1515] [youtube:feed] Check each 'load more' portion for
 unique video ids

---
 youtube_dl/extractor/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9096a2975..1f9940cf5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1621,10 +1621,16 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         # for the video ids doesn't contain an index
         ids = []
         more_widget_html = content_html = page
-
         for page_num in itertools.count(1):
             matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
-            new_ids = orderedSet(matches)
+
+            # 'recommended' feed has infinite 'load more' and each new portion spins
+            # the same videos in (sometimes) slightly different order, so we'll check
+            # for unicity and break when portion has no new videos
+            new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
+            if not new_ids:
+                break
+
             ids.extend(new_ids)
 
             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)

From e9ca615a9872e85a6986061fdf54257244ce1f77 Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 19:57:54 +0400
Subject: [PATCH 0616/1515] New test

---
 youtube_dl/extractor/sportbox.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 695b3ff82..cb1515eff 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -30,10 +30,29 @@ class SportBoxIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
-        }, {
+        }, 
+        {
+            'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',            
+            'md5': 'ff56a598c2cf411a9a38a69709e97079',
+            'info_dict': {
+                'id': '211355',
+                'ext': 'mp4',
+                'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+                'description': '16 детских коллективов приняли участие в суперфинале турнира «Поле славы боевой».',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1426237001,
+                'upload_date': '20150313',
+                'duration': 292,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        }, 
+        {
             'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
             'only_matching': True,
-        }
+        },
     ]
 
     def _real_extract(self, url):

From 34fe5a94baf9e7ea437de68621a5fa73780c0f17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 18:42:59 +0200
Subject: [PATCH 0617/1515] [gamespot] Add support for videos that don't use
 'f4m_stream' (fixes #5707)

---
 youtube_dl/extractor/gamespot.py | 60 +++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 47373e215..5927455f6 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class GameSpotIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
         'info_dict': {
@@ -23,8 +23,16 @@ class GameSpotIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Arma 3 - Community Guide: SITREP I',
             'description': 'Check out this video where some of the basics of Arma 3 is explained.',
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.gamespot.com/videos/the-witcher-3-wild-hunt-xbox-one-now-playing/2300-6424837/',
+        'info_dict': {
+            'id': 'gs-2300-6424837',
+            'ext': 'flv',
+            'title': 'The Witcher 3: Wild Hunt [Xbox ONE]  - Now Playing',
+            'description': 'Join us as we take a look at the early hours of The Witcher 3: Wild Hunt and more.',
+        },
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
@@ -32,25 +40,37 @@ class GameSpotIE(InfoExtractor):
         data_video_json = self._search_regex(
             r'data-video=["\'](.*?)["\']', webpage, 'data video')
         data_video = json.loads(unescapeHTML(data_video_json))
+        streams = data_video['videoStreams']
 
-        # Transform the manifest url to a link to the mp4 files
-        # they are used in mobile devices.
-        f4m_url = data_video['videoStreams']['f4m_stream']
-        f4m_path = compat_urlparse.urlparse(f4m_url).path
-        QUALITIES_RE = r'((,\d+)+,?)'
-        qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
-        http_path = f4m_path[1:].split('/', 1)[1]
-        http_template = re.sub(QUALITIES_RE, r'%s', http_path)
-        http_template = http_template.replace('.csmil/manifest.f4m', '')
-        http_template = compat_urlparse.urljoin(
-            'http://video.gamespotcdn.com/', http_template)
         formats = []
-        for q in qualities:
-            formats.append({
-                'url': http_template % q,
-                'ext': 'mp4',
-                'format_id': q,
-            })
+        f4m_url = streams.get('f4m_stream')
+        if f4m_url is not None:
+            # Transform the manifest url to a link to the mp4 files
+            # they are used in mobile devices.
+            f4m_path = compat_urlparse.urlparse(f4m_url).path
+            QUALITIES_RE = r'((,\d+)+,?)'
+            qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
+            http_path = f4m_path[1:].split('/', 1)[1]
+            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
+            http_template = http_template.replace('.csmil/manifest.f4m', '')
+            http_template = compat_urlparse.urljoin(
+                'http://video.gamespotcdn.com/', http_template)
+            for q in qualities:
+                formats.append({
+                    'url': http_template % q,
+                    'ext': 'mp4',
+                    'format_id': q,
+                })
+        else:
+            for quality in ['sd', 'hd']:
+                # It's actually a link to a flv file
+                flv_url = streams.get('f4m_{0}'.format(quality))
+                if flv_url is not None:
+                    formats.append({
+                        'url': flv_url,
+                        'ext': 'flv',
+                        'format_id': quality,
+                    })
 
         return {
             'id': data_video['guid'],

From eeb23eb7ea6953d7e90ccf669cd0e636d10b2b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 18:44:08 +0200
Subject: [PATCH 0618/1515] [gamespot] The protocol is not optional

---
 youtube_dl/extractor/gamespot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 5927455f6..2d33fa7f5 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'http://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',

From 3a7382950b6f498f50173c8813f6cb1db3739277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 22:50:44 +0600
Subject: [PATCH 0619/1515] [sportbox:embed] Add extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/sportbox.py | 138 +++++++++++++++++--------------
 2 files changed, 82 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8ec0c1032..f293bc2a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -502,7 +502,10 @@ from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
 from .sport5 import Sport5IE
-from .sportbox import SportBoxIE
+from .sportbox import (
+    SportBoxIE,
+    SportBoxEmbedIE,
+)
 from .sportdeutschland import SportDeutschlandIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index cb1515eff..10c45eb74 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
     parse_iso8601,
@@ -12,48 +13,30 @@ from ..utils import (
 
 class SportBoxIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
-    _TESTS = [
-        {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',            
-            'md5': 'ff56a598c2cf411a9a38a69709e97079',
-            'info_dict': {
-                'id': '80822',
-                'ext': 'mp4',
-                'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-                'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'timestamp': 1411896237,
-                'upload_date': '20140928',
-                'duration': 4846,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }, 
-        {
-            'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',            
-            'md5': 'ff56a598c2cf411a9a38a69709e97079',
-            'info_dict': {
-                'id': '211355',
-                'ext': 'mp4',
-                'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
-                'description': '16 детских коллективов приняли участие в суперфинале турнира «Поле славы боевой».',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'timestamp': 1426237001,
-                'upload_date': '20150313',
-                'duration': 292,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }, 
-        {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
-            'only_matching': True,
+    _TESTS = [{
+        'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+        'md5': 'ff56a598c2cf411a9a38a69709e97079',
+        'info_dict': {
+            'id': '80822',
+            'ext': 'mp4',
+            'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
+            'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1411896237,
+            'upload_date': '20140928',
+            'duration': 4846,
         },
-    ]
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
+        'only_matching': True,
+    }, {
+        'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -61,39 +44,74 @@ class SportBoxIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        sobj = re.search(r'src="/vdl/player/(?P<media_type>\w+)/(?P<video_id>\d+)"', webpage)
-        if (sobj):
-            video_id = sobj.group('video_id')
-            media_type = sobj.group('media_type')
-        else:
-            raise RegexNotFoundError('Unable to extract video_id')
-
-        player = self._download_webpage(
-            'http://news.sportbox.ru/vdl/player/%s/%s' % (media_type, video_id),
-            display_id, 'Downloading player webpage')
-
-        hls = self._search_regex(
-            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]+([^\"]+)['\"]+", player, 'hls file')
-
-        formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
+        player = self._search_regex(
+            r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
 
         title = self._html_search_regex(
             r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r'(?s)<div itemprop="description">(.+?)</div>', webpage, 'description', fatal=False)
+            r'(?s)<div itemprop="description">(.+?)</div>',
+            webpage, 'description', fatal=False)
         thumbnail = self._og_search_thumbnail(webpage)
         timestamp = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
+            r'<span itemprop="uploadDate">([^<]+)</span>',
+            webpage, 'timestamp', fatal=False))
         duration = parse_duration(self._html_search_regex(
-            r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
+            r'<meta itemprop="duration" content="PT([^"]+)">',
+            webpage, 'duration', fatal=False))
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'url': compat_urlparse.urljoin(url, '/%s' % player),
             'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
             'duration': duration,
+        }
+
+
+class SportBoxEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://news\.sportbox\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://news.sportbox.ru/vdl/player/ci/211355',
+        'info_dict': {
+            'id': '211355',
+            'ext': 'mp4',
+            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://news.sportbox.ru/vdl/player?nid=370908&only_player=1&autostart=false&playeri=2&height=340&width=580',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        hls = self._search_regex(
+            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]([^'\"]+)['\"]",
+            webpage, 'hls file')
+
+        formats = self._extract_m3u8_formats(hls, video_id, 'mp4')
+
+        title = self._search_regex(
+            r'sportboxPlayer\.node_title\s*=\s*"([^"]+)"', webpage, 'title')
+
+        thumbnail = self._search_regex(
+            r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
         }

From e8cfacae3710c2c225488e4b2d41b84268217a55 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 15 May 2015 17:57:32 +0100
Subject: [PATCH 0620/1515] [CBSNewsIE] Relax thumbnail regex so test passes

---
 youtube_dl/extractor/cbsnews.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7e47960ab..52e61d85b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -32,7 +32,7 @@ class CBSNewsIE(InfoExtractor):
                 'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
                 'ext': 'flv',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
-                'thumbnail': 'http://cbsnews2.cbsistatic.com/hub/i/r/2014/04/04/0c9fbc66-576b-41ca-8069-02d122060dd2/thumbnail/140x90/6dad7a502f88875ceac38202984b6d58/en-0404-werner-replace-640x360.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
             },
             'params': {

From 1436a6835e0f3489a4c37cca3da5087567b68158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:08:44 +0600
Subject: [PATCH 0621/1515] [sportbox:embed] Add `_extract_urls`

---
 youtube_dl/extractor/sportbox.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 10c45eb74..a869a1b25 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -91,6 +91,12 @@ class SportBoxEmbedIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src="(https?://news\.sportbox\.ru/vdl/player[^"]+)"',
+            webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From ef28a6cb26630f8f198a72eee34a2b5c8bd2f802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:09:10 +0600
Subject: [PATCH 0622/1515] [sportbox:embed] Relax thumbnail

---
 youtube_dl/extractor/sportbox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index a869a1b25..8686f9d11 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -113,7 +113,7 @@ class SportBoxEmbedIE(InfoExtractor):
 
         thumbnail = self._search_regex(
             r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
+            webpage, 'thumbnail', default=None)
 
         return {
             'id': video_id,

From d40a3b5b55973d7ed65538179b71990c1828845a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:09:34 +0600
Subject: [PATCH 0623/1515] [generic] Add support for sportbox embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3d756e848..9230c3bb0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,6 +32,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
+from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
@@ -1229,6 +1230,11 @@ class GenericIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
+        # Look for embedded SportBox player
+        sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
+        if sportbox_urls:
+            return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
+
         # Look for embedded TED player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)

From b827a6015c145d67a4d4e9ea38aa54ebe347d3fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:18:21 +0600
Subject: [PATCH 0624/1515] [generic] Add test for sportbox embeds

---
 youtube_dl/extractor/generic.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9230c3bb0..610e33091 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -225,6 +225,37 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # SportBox embed
+        {
+            'url': 'http://www.vestifinance.ru/articles/25753',
+            'info_dict': {
+                'id': '25753',
+                'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '370908',
+                    'title': 'Госзаказ. День 3',
+                    'ext': 'mp4',
+                }
+            }, {
+                'info_dict': {
+                    'id': '370905',
+                    'title': 'Госзаказ. День 2',
+                    'ext': 'mp4',
+                }
+            }, {
+                'info_dict': {
+                    'id': '370902',
+                    'title': 'Госзаказ. День 1',
+                    'ext': 'mp4',
+                }
+            }],
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        },
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',

From 25c3a7348f7971c0af32dcea2d7fd57bd5c63f05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:23:51 +0600
Subject: [PATCH 0625/1515] [generic] Fix typo

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 610e33091..9a7b0d25d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1425,7 +1425,7 @@ class GenericIE(InfoExtractor):
         # Look for Senate ISVP iframe
         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
         if senate_isvp_url:
-            return self.url_result(surl, 'SenateISVP')
+            return self.url_result(senate_isvp_url, 'SenateISVP')
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):

From 70d0d43b5eeff04b41b089e499401e38c115e456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:32:25 +0600
Subject: [PATCH 0626/1515] [rts] Check formats (Closes #5711)

---
 youtube_dl/extractor/rts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index d0981115d..9fbe239d8 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -190,6 +190,7 @@ class RTSIE(InfoExtractor):
                 'tbr': media['rate'] or extract_bitrate(media['url']),
             } for media in info['media'] if media.get('rate')])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         return {

From 0d7f03642976e7859e290b06db41d20a4bfd3a38 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 May 2015 15:43:13 +0800
Subject: [PATCH 0627/1515] [viki] Add support for shows

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/viki.py     | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f293bc2a4..cb6635610 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -639,7 +639,10 @@ from .vine import (
     VineIE,
     VineUserIE,
 )
-from .viki import VikiIE
+from .viki import (
+    VikiIE,
+    VikiShowIE,
+)
 from .vk import (
     VKIE,
     VKUserVideosIE,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index cf6af1e5c..4d185c0e6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -145,3 +145,36 @@ class VikiIE(InfoExtractor):
                 'ext': 'vtt',
             }]
         return res
+
+
+class VikiShowIE(InfoExtractor):
+    IE_NAME = 'viki:show'
+    _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _TESTS = [{
+        'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
+        'info_dict': {
+            'id': '50c',
+            'title': 'Boys Over Flowers',
+            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
+        },
+        'playlist_count': 25,
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        show_page = self._download_webpage(url, show_id, 'Download show page')
+
+        title = self._og_search_title(show_page)
+        description = self._og_search_description(show_page)
+
+        show_json = self._download_json(
+            'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
+            show_id, note='Retrieve show json', errnote='Unable to get show json'
+        )
+        entries = []
+        for video in show_json['response']:
+            video_id = video['id']
+            entries.append(self.url_result(
+                'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+
+        return self.playlist_result(entries, show_id, title, description)

From 2f3bdab2b90c6695c0a478f352967b0c9da4f23f Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 May 2015 15:56:37 +0800
Subject: [PATCH 0628/1515] [viki] Fix code format

---
 youtube_dl/extractor/viki.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 4d185c0e6..40a73f561 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -169,8 +169,7 @@ class VikiShowIE(InfoExtractor):
 
         show_json = self._download_json(
             'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
-            show_id, note='Retrieve show json', errnote='Unable to get show json'
-        )
+            show_id, note='Retrieve show json', errnote='Unable to get show json')
         entries = []
         for video in show_json['response']:
             video_id = video['id']

From 1c97b0a777f52c520587e93e7e61721fa6195977 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 20:00:40 +0600
Subject: [PATCH 0629/1515] [ooyala:external] Add extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/ooyala.py   | 221 +++++++++++++++++++------------
 2 files changed, 137 insertions(+), 89 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f293bc2a4..1731f4fb2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -376,7 +376,10 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
-from .ooyala import OoyalaIE
+from .ooyala import (
+    OoyalaIE,
+    OoyalaExternalIE,
+)
 from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index c0e6d643d..9f4fe5b29 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -12,7 +12,100 @@ from ..utils import (
 )
 
 
-class OoyalaIE(InfoExtractor):
+class OoyalaBaseIE(InfoExtractor):
+
+    def _extract_result(self, info, more_info):
+        embedCode = info['embedCode']
+        video_url = info.get('ipad_url') or info['url']
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': 'mp4',
+            }]
+
+        return {
+            'id': embedCode,
+            'title': unescapeHTML(info['title']),
+            'formats': formats,
+            'description': unescapeHTML(more_info['description']),
+            'thumbnail': more_info['promo'],
+        }
+
+    def _extract(self, player_url, video_id):
+        player = self._download_webpage(player_url, video_id)
+        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
+                                        player, 'mobile player url')
+        # Looks like some videos are only available for particular devices
+        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
+        # is only available for ipad)
+        # Working around with fetching URLs for all the devices found starting with 'unknown'
+        # until we succeed or eventually fail for each device.
+        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
+        devices.remove('unknown')
+        devices.insert(0, 'unknown')
+        for device in devices:
+            mobile_player = self._download_webpage(
+                '%s&device=%s' % (mobile_url, device), video_id,
+                'Downloading mobile player JS for %s device' % device)
+            videos_info = self._search_regex(
+                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
+                mobile_player, 'info', fatal=False, default=None)
+            if videos_info:
+                break
+
+        if not videos_info:
+            formats = []
+            auth_data = self._download_json(
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                video_id)
+
+            cur_auth_data = auth_data['authorization_data'][video_id]
+
+            for stream in cur_auth_data['streams']:
+                formats.append({
+                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
+                    'ext': stream.get('delivery_type'),
+                    'format': stream.get('video_codec'),
+                    'format_id': stream.get('profile'),
+                    'width': int_or_none(stream.get('width')),
+                    'height': int_or_none(stream.get('height')),
+                    'abr': int_or_none(stream.get('audio_bitrate')),
+                    'vbr': int_or_none(stream.get('video_bitrate')),
+                })
+            if formats:
+                return {
+                    'id': video_id,
+                    'formats': formats,
+                    'title': 'Ooyala video',
+                }
+
+            if not cur_auth_data['authorized']:
+                raise ExtractorError(cur_auth_data['message'], expected=True)
+
+        if not videos_info:
+            raise ExtractorError('Unable to extract info')
+        videos_info = videos_info.replace('\\"', '"')
+        videos_more_info = self._search_regex(
+            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
+        videos_info = json.loads(videos_info)
+        videos_more_info = json.loads(videos_more_info)
+
+        if videos_more_info.get('lineup'):
+            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': unescapeHTML(videos_more_info['title']),
+                'entries': videos,
+            }
+        else:
+            return self._extract_result(videos_info[0], videos_more_info)
+
+
+class OoyalaIE(OoyalaBaseIE):
     _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'
 
     _TESTS = [
@@ -57,95 +150,47 @@ class OoyalaIE(InfoExtractor):
         return cls.url_result(cls._url_for_embed_code(embed_code),
                               ie=cls.ie_key())
 
-    def _extract_result(self, info, more_info):
-        embedCode = info['embedCode']
-        video_url = info.get('ipad_url') or info['url']
+    def _real_extract(self, url):
+        embed_code = self._match_id(url)
+        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
+        return self._extract(player_url, embed_code)
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': 'mp4',
-            }]
 
-        return {
-            'id': embedCode,
-            'title': unescapeHTML(info['title']),
-            'formats': formats,
-            'description': unescapeHTML(more_info['description']),
-            'thumbnail': more_info['promo'],
-        }
+class OoyalaExternalIE(OoyalaBaseIE):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        ooyalaexternal:|
+                        https?://.+?\.ooyala\.com/.*?\bexternalId=
+                    )
+                    (?P<partner_id>[^:]+)
+                    :
+                    (?P<id>.+)
+                    (?:
+                        :|
+                        .*?&pcode=
+                    )
+                    (?P<pcode>.+?)
+                    (&|$)
+                    '''
+
+    _TEST = {
+        'url': 'https://player.ooyala.com/player.js?externalId=espn:10365079&pcode=1kNG061cgaoolOncv54OAO1ceO-I&adSetCode=91cDU6NuXTGKz3OdjOxFdAgJVtQcKJnI&callback=handleEvents&hasModuleParams=1&height=968&playerBrandingId=7af3bd04449c444c964f347f11873075&targetReplaceId=videoPlayer&width=1656&wmode=opaque&allowScriptAccess=always',
+        'info_dict': {
+            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
+            'ext': 'mp4',
+            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'description': '',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        embedCode = mobj.group('id')
-        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embedCode
-        player = self._download_webpage(player_url, embedCode)
-        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
-                                        player, 'mobile player url')
-        # Looks like some videos are only available for particular devices
-        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
-        # is only available for ipad)
-        # Working around with fetching URLs for all the devices found starting with 'unknown'
-        # until we succeed or eventually fail for each device.
-        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
-        devices.remove('unknown')
-        devices.insert(0, 'unknown')
-        for device in devices:
-            mobile_player = self._download_webpage(
-                '%s&device=%s' % (mobile_url, device), embedCode,
-                'Downloading mobile player JS for %s device' % device)
-            videos_info = self._search_regex(
-                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
-                mobile_player, 'info', fatal=False, default=None)
-            if videos_info:
-                break
-
-        if not videos_info:
-            formats = []
-            auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
-                embedCode)
-
-            cur_auth_data = auth_data['authorization_data'][embedCode]
-
-            for stream in cur_auth_data['streams']:
-                formats.append({
-                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
-                    'ext': stream.get('delivery_type'),
-                    'format': stream.get('video_codec'),
-                    'format_id': stream.get('profile'),
-                    'width': int_or_none(stream.get('width')),
-                    'height': int_or_none(stream.get('height')),
-                    'abr': int_or_none(stream.get('audio_bitrate')),
-                    'vbr': int_or_none(stream.get('video_bitrate')),
-                })
-            if formats:
-                return {
-                    'id': embedCode,
-                    'formats': formats,
-                    'title': 'Ooyala video',
-                }
-
-            if not cur_auth_data['authorized']:
-                raise ExtractorError(cur_auth_data['message'], expected=True)
-
-        if not videos_info:
-            raise ExtractorError('Unable to extract info')
-        videos_info = videos_info.replace('\\"', '"')
-        videos_more_info = self._search_regex(
-            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
-        videos_info = json.loads(videos_info)
-        videos_more_info = json.loads(videos_more_info)
-
-        if videos_more_info.get('lineup'):
-            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
-            return {
-                '_type': 'playlist',
-                'id': embedCode,
-                'title': unescapeHTML(videos_more_info['title']),
-                'entries': videos,
-            }
-        else:
-            return self._extract_result(videos_info[0], videos_more_info)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+        pcode = mobj.group('pcode')
+        player_url = 'http://player.ooyala.com/player.js?externalId=%s:%s&pcode=%s' % (partner_id, video_id, pcode)
+        return self._extract(player_url, video_id)

From 9354a5fad4521687eb9f08c1a42848621857400b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 20:15:31 +0600
Subject: [PATCH 0630/1515] [ooyala] Fix unresolved reference

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 9f4fe5b29..a262a9f6d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -59,7 +59,7 @@ class OoyalaBaseIE(InfoExtractor):
         if not videos_info:
             formats = []
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (video_id, video_id),
                 video_id)
 
             cur_auth_data = auth_data['authorization_data'][video_id]

From ef2dcbe4adce4478d409397faaae7ec6453ecf7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 21:07:29 +0600
Subject: [PATCH 0631/1515] [sbs] Fix extraction (Closes #5725)

---
 youtube_dl/extractor/sbs.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index b8775c2f9..3073e5e86 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -33,16 +33,18 @@ class SBSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        release_urls_json = js_to_json(self._search_regex(
+        player = self._search_regex(
             r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
-            webpage, ''))
-        release_urls = json.loads(release_urls_json)
-        theplatform_url = (
-            release_urls.get('progressive') or release_urls.get('standard'))
+            webpage, 'player')
+        player = re.sub(r"'\s*\+\s*[\da-zA-Z_]+\s*\+\s*'", '', player)
+
+        release_urls = self._parse_json(js_to_json(player), video_id)
+
+        theplatform_url = release_urls.get('progressive') or release_urls['standard']
 
         title = remove_end(self._og_search_title(webpage), ' (The Feed)')
         description = self._html_search_meta('description', webpage)
@@ -52,7 +54,6 @@ class SBSIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': theplatform_url,
-
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 7e760fc18897663db7c0717434e28a8cca9f3810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 21:14:19 +0600
Subject: [PATCH 0632/1515] [espn] Add extractor (#4396)

Unfinished
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/espn.py     | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/espn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1731f4fb2..6b19eb6f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -141,6 +141,7 @@ from .engadget import EngadgetIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
+from .espn import ESPNIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
new file mode 100644
index 000000000..e6f8f0337
--- /dev/null
+++ b/youtube_dl/extractor/espn.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ESPNIE(InfoExtractor):
+    _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
+    _WORKING = False
+    _TESTS = [{
+        'url': 'http://espn.go.com/video/clip?id=10365079',
+        'info_dict': {
+            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
+            'ext': 'mp4',
+            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'description': '',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/nba/recap?gameId=400793786',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'class="video-play-button"[^>]+data-id="(\d+)',
+            webpage, 'video id')
+
+        player = self._download_webpage(
+            'https://espn.go.com/video/iframe/twitter/?id=%s' % video_id, video_id)
+
+        pcode = self._search_regex(
+            r'["\']pcode=([^"\']+)["\']', player, 'pcode')
+
+        return self.url_result(
+            'ooyalaexternal:espn:%s:%s' % (video_id, pcode),
+            'OoyalaExternal')

From 4d52f2eb7f4b16ea5491f20abf0b29a1fcb24a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 16 May 2015 18:38:28 +0200
Subject: [PATCH 0633/1515] [sbs] Remove unused import

---
 youtube_dl/extractor/sbs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 3073e5e86..d4bd1a0d7 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import json
 import re
 from .common import InfoExtractor
 from ..utils import (

From 1c18de00192d195357989861563cc1fad9256128 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 May 2015 01:38:50 +0800
Subject: [PATCH 0634/1515] [viki] Add proper paging and include clips

---
 youtube_dl/extractor/viki.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 40a73f561..4d477b03c 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -93,7 +93,7 @@ class VikiIE(InfoExtractor):
                     'Video %s is blocked from your location.' % video_id,
                     expected=True)
             else:
-                raise ExtractorError('Viki said: ' + err_msg)
+                raise ExtractorError('Viki said: %s %s' % (err_msg, url))
         mobj = re.search(
             r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
         if not mobj:
@@ -157,7 +157,15 @@ class VikiShowIE(InfoExtractor):
             'title': 'Boys Over Flowers',
             'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
         },
-        'playlist_count': 25,
+        'playlist_count': 70,
+    }, {
+        'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
+        'info_dict': {
+            'id': '1354c',
+            'title': 'Poor Nastya [COMPLETE]',
+            'description': 'md5:05bf5471385aa8b21c18ad450e350525',
+        },
+        'playlist_count': 127,
     }]
 
     def _real_extract(self, url):
@@ -167,13 +175,16 @@ class VikiShowIE(InfoExtractor):
         title = self._og_search_title(show_page)
         description = self._og_search_description(show_page)
 
-        show_json = self._download_json(
-            'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
-            show_id, note='Retrieve show json', errnote='Unable to get show json')
         entries = []
-        for video in show_json['response']:
-            video_id = video['id']
-            entries.append(self.url_result(
-                'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+        for video_type in ['episodes', 'clips']:
+            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=25&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type)
+            while json_url is not None:
+                show_json = self._download_json(
+                    json_url, show_id, note='Retrieve show json', errnote='Unable to get show json')
+                for video in show_json['response']:
+                    video_id = video['id']
+                    entries.append(self.url_result(
+                        'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+                json_url = show_json['pagination']['next']
 
         return self.playlist_result(entries, show_id, title, description)

From baa43cbaf01a575eacb8e1bb39c7200f68c36daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 02:59:35 +0600
Subject: [PATCH 0635/1515] [extractor/common] Relax valid url check verbosity

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 65bb77086..cecf917ff 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -786,8 +786,8 @@ class InfoExtractor(object):
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
-                self.report_warning(
-                    '%s URL is invalid, skipping' % item, video_id)
+                self.to_screen(
+                    '%s: %s URL is invalid, skipping' % (video_id, item))
                 return False
             raise
 

From bc0f937b55aae6ce731d259a7658b0281c2e62ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 03:01:52 +0600
Subject: [PATCH 0636/1515] [tv2] Add extractor (#5724)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tv2.py      | 93 ++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 youtube_dl/extractor/tv2.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6b19eb6f8..fb4f63ca3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -572,6 +572,7 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
+from .tv2 import TV2IE
 from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
new file mode 100644
index 000000000..2dcc0e971
--- /dev/null
+++ b/youtube_dl/extractor/tv2.py
@@ -0,0 +1,93 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
+)
+
+
+class TV2IE(InfoExtractor):
+    _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tv2.no/v/916509/',
+        'md5': '9cb9e3410b18b515d71892f27856e9b1',
+        'info_dict': {
+            'id': '916509',
+            'ext': 'flv',
+            'title': 'Se Gryttens hyllest av Steven Gerrard',
+            'description': 'TV 2 Sportens huspoet tar avskjed med Liverpools kaptein Steven Gerrard.',
+            'timestamp': 1431715610,
+            'upload_date': '20150515',
+            'duration': 156.967,
+            'view_count': int,
+            'categories': list,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        formats = []
+        format_urls = []
+        for protocol in ('HDS', 'HLS'):
+            data = self._download_json(
+                'http://sumo.tv2.no/api/web/asset/%s/play.json?protocol=%s&videoFormat=SMIL+ISMUSP' % (video_id, protocol),
+                video_id, 'Downloading play JSON')['playback']
+            for item in data['items']['item']:
+                video_url = item.get('url')
+                if not video_url or video_url in format_urls:
+                    continue
+                format_id = '%s-%s' % (protocol.lower(), item.get('mediaFormat'))
+                if not self._is_valid_url(video_url, video_id, format_id):
+                    continue
+                format_urls.append(video_url)
+                ext = determine_ext(video_url)
+                if ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        video_url, video_id, f4m_id=format_id))
+                elif ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=format_id))
+                elif ext == 'ism' or video_url.endswith('.ism/Manifest'):
+                    pass
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                        'tbr': int_or_none(item.get('bitrate')),
+                        'filesize': int_or_none(item.get('fileSize')),
+                    })
+        self._sort_formats(formats)
+
+        asset = self._download_json(
+            'http://sumo.tv2.no/api/web/asset/%s.json' % video_id,
+            video_id, 'Downloading metadata JSON')['asset']
+
+        title = asset['title']
+        description = asset.get('description')
+        timestamp = parse_iso8601(asset.get('createTime'))
+        duration = float_or_none(asset.get('accurateDuration') or asset.get('duration'))
+        view_count = int_or_none(asset.get('views'))
+        categories = asset.get('keywords', '').split(',')
+
+        thumbnails = [{
+            'id': thumbnail.get('@type'),
+            'url': thumbnail.get('url'),
+        } for _, thumbnail in asset.get('imageVersions', {}).items()]
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'categories': categories,
+            'formats': formats,
+        }

From 588b82bbf8c90981c54f180eca40e6c743f8f89f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 03:32:53 +0600
Subject: [PATCH 0637/1515] [tv2:article] Add extractor (Closes #5724)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/tv2.py      | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb4f63ca3..6f8c261d5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -572,7 +572,10 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
-from .tv2 import TV2IE
+from .tv2 import (
+    TV2IE,
+    TV2ArticleIE,
+)
 from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 2dcc0e971..fa338b936 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -1,12 +1,15 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
     float_or_none,
     parse_iso8601,
+    remove_end,
 )
 
 
@@ -91,3 +94,33 @@ class TV2IE(InfoExtractor):
             'categories': categories,
             'formats': formats,
         }
+
+
+class TV2ArticleIE(InfoExtractor):
+    _VALID_URL = 'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
+        'info_dict': {
+            'id': '6930542',
+            'title': 'Russen hetses etter pingvintyveri – innrømmer å ha åpnet luken på buret',
+            'description': 'md5:339573779d3eea3542ffe12006190954',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.tv2.no/a/6930542',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.tv2.no/v/%s' % video_id, 'TV2')
+            for video_id in re.findall(r'data-assetid="(\d+)"', webpage)]
+
+        title = remove_end(self._og_search_title(webpage), ' - TV2.no')
+        description = remove_end(self._og_search_description(webpage), ' - TV2.no')
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 8da0e0e94682faa0463f33d991df70a2402b5a86 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 May 2015 06:19:38 +0800
Subject: [PATCH 0638/1515] [viki] Change IE name to channel, better message
 output

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/viki.py     | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cb6635610..21f7b7290 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -641,7 +641,7 @@ from .vine import (
 )
 from .viki import (
     VikiIE,
-    VikiShowIE,
+    VikiChannelIE,
 )
 from .vk import (
     VKIE,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 4d477b03c..9bdbdc3e4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -147,8 +147,8 @@ class VikiIE(InfoExtractor):
         return res
 
 
-class VikiShowIE(InfoExtractor):
-    IE_NAME = 'viki:show'
+class VikiChannelIE(InfoExtractor):
+    IE_NAME = 'viki:channel'
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
@@ -167,6 +167,7 @@ class VikiShowIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
+    _PER_PAGE = 25
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
@@ -177,10 +178,12 @@ class VikiShowIE(InfoExtractor):
 
         entries = []
         for video_type in ['episodes', 'clips']:
-            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=25&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type)
+            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type, self._PER_PAGE)
             while json_url is not None:
                 show_json = self._download_json(
-                    json_url, show_id, note='Retrieve show json', errnote='Unable to get show json')
+                    json_url, show_id,
+                    note='Downloading %s json page #%s' %
+                         (video_type, re.search(r'[?&]page=([0-9]+)', json_url).group(1)))
                 for video in show_json['response']:
                     video_id = video['id']
                     entries.append(self.url_result(

From 725652e9247e1171110b624d748e20fa1c88260e Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sat, 16 May 2015 19:50:58 -0500
Subject: [PATCH 0639/1515] [karrierevideos] add support for
 www.karrierevideos.at (closes #5354)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/karrierevideos.py | 52 ++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/karrierevideos.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb4f63ca3..d131d3ec3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -244,6 +244,7 @@ from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
+from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
new file mode 100644
index 000000000..59d29e845
--- /dev/null
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KarriereVideosIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P<id>[a-z-]+)'
+    _TEST = {
+        'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
+        'info_dict': {
+            'id': 'altenpflegerin',
+            'ext': 'mp4',
+            'title': 'AltenpflegerIn',
+            'thumbnail': 're:^http://.*\.png\?v=[0-9]+',
+            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2'
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        description = self._html_search_regex(
+            r'<div class="leadtext">\n{0,}?\s{0,}<p>(.*?)</p>',
+            webpage, 'description')
+
+        playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
+        playlist = self._download_xml(
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist,
+            video_id)
+
+        namespace = {
+            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
+        }
+
+        item = playlist.find('tracklist/item')
+        streamer = item.find('jwplayer:streamer', namespace).text
+
+        return {
+            'id': video_id,
+            'title': self._html_search_meta('title', webpage),
+            'description': description,
+            'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
+            'protocol': 'rtmp',
+            'url': streamer.replace('rtmpt', 'http'),
+            'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text,
+            'tc_url': streamer,
+            'ext': 'mp4'
+        }

From ba9d16291b8ace3bd412bcfc0c128c047545e509 Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 17 May 2015 03:35:08 -0500
Subject: [PATCH 0640/1515] manually specify namespace

---
 youtube_dl/extractor/karrierevideos.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index 59d29e845..a05e8ab76 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -29,15 +29,13 @@ class KarriereVideosIE(InfoExtractor):
 
         playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
         playlist = self._download_xml(
-            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist,
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist,
             video_id)
 
-        namespace = {
-            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
-        }
+        namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
 
         item = playlist.find('tracklist/item')
-        streamer = item.find('jwplayer:streamer', namespace).text
+        streamer = item.find('{%s}streamer' % namespace).text
 
         return {
             'id': video_id,
@@ -46,7 +44,7 @@ class KarriereVideosIE(InfoExtractor):
             'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
             'protocol': 'rtmp',
             'url': streamer.replace('rtmpt', 'http'),
-            'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text,
+            'play_path': 'mp4:' + item.find('{%s}file' % namespace).text,
             'tc_url': streamer,
             'ext': 'mp4'
         }

From 4a5a898a8fa392d02102672f9767f33a39a73066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 20:56:03 +0600
Subject: [PATCH 0641/1515] [YoutubeDL] Clarify incompatible formats merge
 message

When `-f` is not specified it's misleading to see `You have requested ...` as user did not actually request any formats.
---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5df889945..58b34e087 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1368,7 +1368,7 @@ class YoutubeDL(object):
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
-                                            ' The formats won\'t be merged')
+                                            ' The formats won\'t be merged.')
                     else:
                         postprocessors = [merger]
 
@@ -1395,8 +1395,8 @@ class YoutubeDL(object):
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         info_dict['ext'] = 'mkv'
-                        self.report_warning('You have requested formats incompatible for merge. '
-                                            'The formats will be merged into mkv')
+                        self.report_warning(
+                            'Requested formats are incompatible for merge and will be merged into mkv.')
                     # Ensure filename always has a correct extension for successful merge
                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
                     if os.path.exists(encodeFilename(filename)):

From fc6e75dd57f3497b99def659b3d0f173b195b7d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 May 2015 11:21:09 +0200
Subject: [PATCH 0642/1515] [instagram] Only recognize https urls (fixes #5739)

http urls redirect to them.
---
 youtube_dl/extractor/instagram.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 65f6ca103..b10755788 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -7,9 +7,9 @@ from ..utils import int_or_none
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https?://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
     _TEST = {
-        'url': 'http://instagram.com/p/aye83DjauH/?foo=bar#abc',
+        'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
         'info_dict': {
             'id': 'aye83DjauH',
@@ -41,11 +41,11 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'http://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {
-        'url': 'http://instagram.com/porsche',
+        'url': 'https://instagram.com/porsche',
         'info_dict': {
             'id': 'porsche',
             'title': 'porsche',

From 5bdc520cf19f404247ec2be1ffc1e83449fa2375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:23:05 +0600
Subject: [PATCH 0643/1515] [xminus] Fix extraction

---
 youtube_dl/extractor/xminus.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index 8c6241aed..7c9d8af6f 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -43,7 +43,7 @@ class XMinusIE(InfoExtractor):
             r'minus_track\.dur_sec=\'([0-9]*?)\'',
             webpage, 'duration', fatal=False))
         filesize_approx = parse_filesize(self._html_search_regex(
-            r'<div class="filesize[^"]*"></div>\s*([0-9.]+\s*[a-zA-Z][bB])',
+            r'<div id="finfo"[^>]*>\s*↓\s*([0-9.]+\s*[a-zA-Z][bB])',
             webpage, 'approximate filesize', fatal=False))
         tbr = int_or_none(self._html_search_regex(
             r'<div class="quality[^"]*"></div>\s*([0-9]+)\s*kbps',
@@ -58,7 +58,7 @@ class XMinusIE(InfoExtractor):
             description = re.sub(' *\r *', '\n', description)
 
         enc_token = self._html_search_regex(
-            r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
+            r'minus_track\.s?tkn="(.+?)"', webpage, 'enc_token')
         token = ''.join(
             c if pos == 3 else compat_chr(compat_ord(c) - 1)
             for pos, c in enumerate(reversed(enc_token)))

From 4f514c7e88d2ce8ebe9c2478183e8797cfb2a4c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:29:41 +0600
Subject: [PATCH 0644/1515] [wimp] Fix youtube extraction (Closes #5690)

---
 youtube_dl/extractor/wimp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index d6dec25ca..f69d46a28 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -37,7 +37,8 @@ class WimpIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", webpage, 'video URL')
+            [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],
+            webpage, 'video URL')
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {

From 2328f2fe684f9a9025217c6f149e92a403a4c437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:34:20 +0600
Subject: [PATCH 0645/1515] [vulture] Fix extraction

---
 youtube_dl/extractor/vulture.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vulture.py b/youtube_dl/extractor/vulture.py
index 1eb24a3d6..faa167e65 100644
--- a/youtube_dl/extractor/vulture.py
+++ b/youtube_dl/extractor/vulture.py
@@ -44,7 +44,7 @@ class VultureIE(InfoExtractor):
         query_webpage = self._download_webpage(
             query_url, display_id, note='Downloading query page')
         params_json = self._search_regex(
-            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n,\n',
+            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n?,\n',
             query_webpage,
             'player params')
         params = json.loads(params_json)

From 5d8dcb5342c97b05c037c8c4e80002540db261b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:39:15 +0600
Subject: [PATCH 0646/1515] [vuclip] Fix extraction

---
 youtube_dl/extractor/vuclip.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index c3fde53f5..a6d9b5fee 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -49,7 +49,7 @@ class VuClipIE(InfoExtractor):
         links_code = self._search_regex(
             r'''(?xs)
                 (?:
-                    <img\s+src="/im/play.gif".*?>|
+                    <img\s+src="[^"]*/play.gif".*?>|
                     <!--\ player\ end\ -->\s*</div><!--\ thumb\ end-->
                 )
                 (.*?)

From 484c9d2d5b669220c24c865947c3f65049916b56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:43:54 +0600
Subject: [PATCH 0647/1515] [vier] Fix extraction

---
 youtube_dl/extractor/vier.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 619039e51..15377097e 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -38,11 +38,14 @@ class VierIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_id = self._search_regex(
-            r'"nid"\s*:\s*"(\d+)"', webpage, 'video id')
+            [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
+            webpage, 'video id')
         application = self._search_regex(
-            r'"application"\s*:\s*"([^"]+)"', webpage, 'application', default='vier_vod')
+            [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
+            webpage, 'application', default='vier_vod')
         filename = self._search_regex(
-            r'"filename"\s*:\s*"([^"]+)"', webpage, 'filename')
+            [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
+            webpage, 'filename')
 
         playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
         formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')

From 2aa64b89b3ac8f387d4c0c27ce7de64bc0ff68de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 May 2015 17:58:53 +0200
Subject: [PATCH 0648/1515] tox: Pass HOME environment variable

Since version 2.0 it only passes a limited set of variables and we need HOME for the tests
---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 00c6e00e3..cd805fe8a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,6 +4,8 @@ envlist = py26,py27,py33,py34
 deps =
    nose
    coverage
+# We need a valid $HOME for test_compat_expanduser
+passenv = HOME
 defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
     --exclude test_subtitles.py --exclude test_write_annotations.py
     --exclude test_youtube_lists.py

From 1b0427e6c433c0b6db5e210db6e3173e19e702ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 00:45:01 +0800
Subject: [PATCH 0649/1515] [utils] Support TTML without default namespace

In a strict sense such TTML is invalid, but Yahoo uses it.
---
 test/test_utils.py  | 15 +++++++++++++++
 youtube_dl/utils.py |  9 ++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index b40107037..e13e11b59 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -621,6 +621,21 @@ Line
 '''
         self.assertEqual(dfxp2srt(dfxp_data), srt_data)
 
+        dfxp_data_no_default_namespace = '''<?xml version="1.0" encoding="UTF-8"?>
+            <tt xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">The first line</p>
+                </div>
+            </body>
+            </tt>'''
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+The first line
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ed9ed9ed6..507f07383 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1848,9 +1848,9 @@ def dfxp2srt(dfxp_data):
         out = str_or_empty(node.text)
 
         for child in node:
-            if child.tag == _x('ttml:br'):
+            if child.tag in (_x('ttml:br'), 'br'):
                 out += '\n' + str_or_empty(child.tail)
-            elif child.tag == _x('ttml:span'):
+            elif child.tag in (_x('ttml:span'), 'span'):
                 out += str_or_empty(parse_node(child))
             else:
                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
@@ -1859,7 +1859,10 @@ def dfxp2srt(dfxp_data):
 
     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p'))
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
+
+    if not paras:
+        raise ValueError('Invalid dfxp/TTML subtitle')
 
     for para, index in zip(paras, itertools.count(1)):
         begin_time = parse_dfxp_time_expr(para.attrib['begin'])

From ecee5724110847b832a6074c66ca4a63758100f4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 00:50:24 +0800
Subject: [PATCH 0650/1515] [yahoo] Add support for closed captions (closes
 #5714)

---
 youtube_dl/extractor/yahoo.py | 18 ++++++++++++++++++
 youtube_dl/utils.py           |  1 +
 2 files changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index bf4e659ac..f9afbdbab 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -15,6 +15,7 @@ from ..utils import (
     unescapeHTML,
     ExtractorError,
     int_or_none,
+    mimetype2ext,
 )
 
 from .nbc import NBCSportsVPlayerIE
@@ -236,6 +237,22 @@ class YahooIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        closed_captions = self._html_search_regex(
+            r'"closedcaptions":(\[[^\]]+\])', webpage, 'closed captions',
+            default='[]')
+
+        cc_json = self._parse_json(closed_captions, video_id, fatal=False)
+        subtitles = {}
+        if cc_json:
+            for closed_caption in cc_json:
+                lang = closed_caption['lang']
+                if lang not in subtitles:
+                    subtitles[lang] = []
+                subtitles[lang].append({
+                    'url': closed_caption['url'],
+                    'ext': mimetype2ext(closed_caption['content_type']),
+                })
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -244,6 +261,7 @@ class YahooIE(InfoExtractor):
             'description': clean_html(meta['description']),
             'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
             'duration': int_or_none(meta.get('duration')),
+            'subtitles': subtitles,
         }
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 507f07383..52d198fa3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1665,6 +1665,7 @@ def mimetype2ext(mt):
     return {
         'x-ms-wmv': 'wmv',
         'x-mp4-fragmented': 'mp4',
+        'ttml+xml': 'ttml',
     }.get(res, res)
 
 

From b813d8caf1b23821036b77b851e42ba0a0ad35a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 01:01:42 +0800
Subject: [PATCH 0651/1515] [qqmusic] Unescape '\\n' in description (#5705)

---
 youtube_dl/extractor/qqmusic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 13113820b..b540033e2 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -26,7 +26,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
-            'description': 'md5:4348ff1dd24036906baa7b6f973f8d30',
+            'description': 'md5:d327722d0361576fde558f1ac68a7065',
         }
     }]
 
@@ -60,6 +60,8 @@ class QQMusicIE(InfoExtractor):
         lrc_content = self._html_search_regex(
             r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)</div>',
             detail_info_page, 'LRC lyrics', default=None)
+        if lrc_content:
+            lrc_content = lrc_content.replace('\\n', '\n')
 
         guid = self.m_r_get_ruin()
 

From d9d747a06ab3b4c36c6063074ffb42aeb185431f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 May 2015 21:28:41 +0600
Subject: [PATCH 0652/1515] [ultimedia] Fix extraction

---
 youtube_dl/extractor/ultimedia.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 96c809eaf..c4751050e 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     ExtractorError,
     qualities,
@@ -44,9 +45,9 @@ class UltimediaIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        deliver_url = self._search_regex(
-            r'<iframe[^>]+src="(https?://(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
-            webpage, 'deliver URL')
+        deliver_url = self._proto_relative_url(self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
+            webpage, 'deliver URL'), compat_urllib_parse_urlparse(url).scheme + ':')
 
         deliver_page = self._download_webpage(
             deliver_url, video_id, 'Downloading iframe page')
@@ -57,7 +58,8 @@ class UltimediaIE(InfoExtractor):
 
         player = self._parse_json(
             self._search_regex(
-                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),
+                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on",
+                deliver_page, 'player'),
             video_id)
 
         quality = qualities(['flash', 'html5'])

From f670ef1c8ebe0329a68b3a3d5c2b7e07ae5c9425 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 13:51:43 +0800
Subject: [PATCH 0653/1515] [dramafever] Add new extractor for dramafever.com

---
 youtube_dl/extractor/__init__.py   |   4 +
 youtube_dl/extractor/dramafever.py | 131 +++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 youtube_dl/extractor/dramafever.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f8c261d5..ca857a75f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -112,6 +112,10 @@ from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
+from .dramafever import (
+    DramaFeverIE,
+    DramaFeverSeriesIE,
+)
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
new file mode 100644
index 000000000..8fac99cc5
--- /dev/null
+++ b/youtube_dl/extractor/dramafever.py
@@ -0,0 +1,131 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class DramaFeverIE(InfoExtractor):
+    IE_NAME = 'dramafever'
+    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)/'
+    _TESTS = [{
+        'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
+        'info_dict': {
+            'id': '4512.1',
+            'ext': 'flv',
+            'title': 'Cooking with Shin 4512.1',
+            'upload_date': '20140702',
+            'description': 'Served at all special occasions and featured in the hit drama Heirs, Shin cooks Red Bean Rice.',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url).replace("/", ".")
+
+        consumer_secret = self._get_consumer_secret(video_id)
+
+        ep_json = self._download_json(
+            "http://www.dramafever.com/amp/episode/feed.json?guid=%s" % video_id,
+            video_id, note='Downloading episode metadata',
+            errnote="Video may not be available for your location")["channel"]["item"]
+
+        title = ep_json["media-group"]["media-title"]
+        description = ep_json["media-group"]["media-description"]
+        thumbnail = ep_json["media-group"]["media-thumbnail"]["@attributes"]["url"]
+        duration = int(ep_json["media-group"]["media-content"][0]["@attributes"]["duration"])
+        mobj = re.match(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", ep_json["pubDate"])
+        upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) if mobj is not None else None
+
+        formats = []
+        for vid_format in ep_json["media-group"]["media-content"]:
+            src = vid_format["@attributes"]["url"]
+            if '.f4m' in src:
+                formats.extend(self._extract_f4m_formats(src, video_id))
+
+        self._sort_formats(formats)
+        video_subtitles = self.extract_subtitles(video_id, consumer_secret)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': video_subtitles,
+        }
+
+    def _get_consumer_secret(self, video_id):
+        df_js = self._download_webpage(
+            "http://www.dramafever.com/static/126960d/v2/js/plugins/jquery.threadedcomments.js", video_id)
+        return self._search_regex(r"'cs': '([0-9a-zA-Z]+)'", df_js, "cs")
+
+    def _get_episodes(self, series_id, consumer_secret, episode_filter=None):
+        _PAGE_SIZE = 60
+
+        curr_page = 1
+        max_pages = curr_page + 1
+        results = []
+        while max_pages >= curr_page:
+            page_url = "http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d" % \
+                       (consumer_secret, series_id, _PAGE_SIZE, curr_page)
+            series = self._download_json(
+                page_url, series_id, note="Downloading series json page #%d" % curr_page)
+            max_pages = series['num_pages']
+            results.extend([ep for ep in series['value'] if episode_filter is None or episode_filter(ep)])
+            curr_page += 1
+        return results
+
+    def _get_subtitles(self, video_id, consumer_secret):
+
+        def match_episode(ep):
+            return ep['guid'] == video_id
+
+        res = None
+        info = self._get_episodes(
+            video_id.split(".")[0], consumer_secret, episode_filter=match_episode)
+        if len(info) == 1 and info[0]['subfile'] != '':
+            res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
+        return res
+
+
+class DramaFeverSeriesIE(DramaFeverIE):
+    IE_NAME = 'dramafever:series'
+    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)/\d*[a-zA-Z_][a-zA-Z0-9_]*/'
+    _TESTS = [{
+        'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
+        'info_dict': {
+            'id': '4512',
+            'title': 'Cooking with Shin',
+            'description': 'Professional chef and cooking instructor Shin Kim takes some of the delicious dishes featured in your favorite dramas and shows you how to make them right at home.',
+        },
+        'playlist_count': 4,
+    }, {
+        'url': 'http://www.dramafever.com/drama/124/IRIS/',
+        'info_dict': {
+            'id': '124',
+            'title': 'IRIS',
+            'description': 'Lee Byung Hun and Kim Tae Hee star in this powerhouse drama and ratings megahit of action, intrigue and romance.',
+        },
+        'playlist_count': 20,
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        consumer_secret = self._get_consumer_secret(series_id)
+
+        series_json = self._download_json(
+            "http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s" % (consumer_secret, series_id),
+            series_id, note='Downloading series metadata')["series"][series_id]
+
+        title = series_json["name"]
+        description = series_json["description_short"]
+
+        episodes = self._get_episodes(series_id, consumer_secret)
+        entries = []
+        for ep in episodes:
+            entries.append(self.url_result(
+                'http://www.dramafever.com%s' % ep['episode_url'], 'DramaFever', ep['guid']))
+        return self.playlist_result(entries, series_id, title, description)

From 051df9ad99d0a29d9eb984970e3e431795b6e445 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 May 2015 14:08:23 +0800
Subject: [PATCH 0654/1515] [letv/sohu] Skip tests relying on external proxies

The proxy is currently broken. See #5655 and zhuzhuor/Unblock-Youku#427
---
 youtube_dl/extractor/letv.py | 4 +---
 youtube_dl/extractor/sohu.py | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 1484ac0d2..da896caf1 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -50,9 +50,7 @@ class LetvIE(InfoExtractor):
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
-        'params': {
-            'cn_verification_proxy': 'http://proxy.uku.im:8888'
-        },
+        'skip': 'Only available in China',
     }]
 
     @staticmethod
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index eab4adfca..29bd9ce6f 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -23,9 +23,7 @@ class SohuIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'MV：Far East Movement《The Illest》',
         },
-        'params': {
-            'cn_verification_proxy': 'proxy.uku.im:8888'
-        }
+        'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
         'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',

From 137597b0ea88a92d174341b44b8f395b8897a2bf Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 15:15:28 +0800
Subject: [PATCH 0655/1515] [dramafever] Streamline code

---
 youtube_dl/extractor/dramafever.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 8fac99cc5..40787ffcd 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -80,12 +80,11 @@ class DramaFeverIE(InfoExtractor):
 
     def _get_subtitles(self, video_id, consumer_secret):
 
-        def match_episode(ep):
-            return ep['guid'] == video_id
-
         res = None
         info = self._get_episodes(
-            video_id.split(".")[0], consumer_secret, episode_filter=match_episode)
+            video_id.split(".")[0], consumer_secret,
+            episode_filter=lambda x: x['guid'] == video_id)
+
         if len(info) == 1 and info[0]['subfile'] != '':
             res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
         return res

From 2632941f327c8b013e5fbc736317fc897876ab73 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 15:53:45 +0800
Subject: [PATCH 0656/1515] [soompi] Add new extractor for tv.soompi.com

---
 youtube_dl/extractor/__init__.py |   4 +
 youtube_dl/extractor/soompi.py   | 130 +++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 youtube_dl/extractor/soompi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f8c261d5..2a5cf9547 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -482,6 +482,10 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
+from .soompi import (
+    SoompiIE,
+    SoompiShowIE,
+)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
new file mode 100644
index 000000000..5ecf40b7f
--- /dev/null
+++ b/youtube_dl/extractor/soompi.py
@@ -0,0 +1,130 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+import base64
+import xml.etree.ElementTree
+
+# Soompi uses the same subtitle encryption as crunchyroll
+from .crunchyroll import CrunchyrollIE
+
+
+class SoompiIE(CrunchyrollIE):
+    IE_NAME = 'soompi'
+    _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://tv.soompi.com/en/watch/23363',
+        'info_dict': {
+            'id': '23363',
+            'ext': 'mp4',
+            'title': 'Liar Game CM1',
+            'description': '15sec'
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _get_episodes(self, webpage, episode_filter=None):
+        episodes = json.loads(
+            self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
+        return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
+
+    def _get_subtitles(self, video_id, show_format_xml):
+        subtitles = {}
+        subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
+        subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
+
+        sub_langs = {}
+        for i in subtitle_info_nodes:
+            sub_langs[i.attrib["id"]] = i.attrib["title"]
+
+        for s in subtitle_nodes:
+            lang_code = sub_langs.get(s.attrib["id"], None)
+            if lang_code is None:
+                continue
+
+            sub_id = int(s.attrib["id"])
+            iv = base64.b64decode(s.find("iv").text)
+            data = base64.b64decode(s.find("data").text)
+            subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
+            sub_root = xml.etree.ElementTree.fromstring(subtitle)
+
+            subtitles[lang_code] = [{
+                'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
+            }, {
+                'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
+            }]
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, video_id, note="Downloading episode page",
+            errnote="Video may not be available for your location")
+        vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage)
+
+        show_meta = json.loads(
+            self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta"))
+        episodes = self._get_episodes(
+            webpage, episode_filter=lambda x: x['id'] == video_id)
+
+        title = episodes[0]["name"]
+        description = episodes[0]["description"]
+        duration = int(episodes[0]["duration"])
+        slug = show_meta["slug"]
+
+        formats = []
+        show_format_xml = None
+        for vf in vid_formats:
+            show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \
+                              % (slug, video_id, vf)
+            show_format_xml = self._download_xml(
+                show_format_url, video_id, note="Downloading q%s show xml" % vf)
+            avail_formats = self._extract_m3u8_formats(
+                show_format_xml.find('./{default}preload/stream_info/file').text,
+                video_id, ext="mp4", m3u8_id=vf, preference=int(vf))
+            formats.extend(avail_formats)
+        self._sort_formats(formats)
+
+        subtitles = self.extract_subtitles(video_id, show_format_xml)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': subtitles
+        }
+
+
+class SoompiShowIE(SoompiIE):
+    IE_NAME = 'soompi:show'
+    _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
+    _TESTS = [{
+        'url': 'http://tv.soompi.com/en/shows/liar-game',
+        'info_dict': {
+            'id': 'liar-game',
+            'title': 'Liar Game',
+            'description': 'md5:52c02bce0c1a622a95823591d0589b66',
+        },
+        'playlist_count': 14,
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, show_id, note="Downloading show page")
+        title = self._og_search_title(webpage).replace("SoompiTV | ", "")
+        description = self._og_search_description(webpage)
+
+        episodes = self._get_episodes(webpage)
+        entries = []
+        for ep in episodes:
+            entries.append(self.url_result(
+                'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
+
+        return self.playlist_result(entries, show_id, title, description)

From 0b9f7cd074786abafcd35b26db4ecb4d92814393 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 May 2015 10:01:48 +0200
Subject: [PATCH 0657/1515] release 2015.05.20

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 43fbe8b1d..a4879bd9a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -142,6 +142,7 @@
  - **Eporner**
  - **EroProfile**
  - **Escapist**
+ - **ESPN** (Currently broken)
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -338,6 +339,7 @@
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
+ - **OoyalaExternal**
  - **OpenFilm**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
@@ -451,6 +453,7 @@
  - **Spike**
  - **Sport5**
  - **SportBox**
+ - **SportBoxEmbed**
  - **SportDeutschland**
  - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
@@ -510,6 +513,8 @@
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
+ - **TV2**
+ - **TV2Article**
  - **TV4**: tv4.se and tv4play.se
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 38f00bc9b..b33385153 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.15'
+__version__ = '2015.05.20'

From 5137adb94dcce98a3c14fb3892c5c72f70ff34ea Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 16:16:10 +0800
Subject: [PATCH 0658/1515] [soompi] Switch to non-geoblocked test video

---
 youtube_dl/extractor/soompi.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
index 5ecf40b7f..4726872dc 100644
--- a/youtube_dl/extractor/soompi.py
+++ b/youtube_dl/extractor/soompi.py
@@ -14,12 +14,12 @@ class SoompiIE(CrunchyrollIE):
     IE_NAME = 'soompi'
     _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://tv.soompi.com/en/watch/23363',
+        'url': 'http://tv.soompi.com/en/watch/29235',
         'info_dict': {
-            'id': '23363',
+            'id': '29235',
             'ext': 'mp4',
-            'title': 'Liar Game CM1',
-            'description': '15sec'
+            'title': 'Episode 1096',
+            'description': '2015-05-20'
         },
         'params': {
             'skip_download': True,

From b0d619fde2b187f2b36b077a1eb11d766429f88c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 May 2015 21:28:04 +0600
Subject: [PATCH 0659/1515] [viki:channel] Extract title from JSON

---
 youtube_dl/extractor/viki.py | 40 +++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 9bdbdc3e4..fc585c299 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -23,7 +23,7 @@ class VikiIE(InfoExtractor):
     # iPad2
     _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
 
-    _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -149,7 +149,7 @@ class VikiIE(InfoExtractor):
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {
@@ -167,27 +167,35 @@ class VikiChannelIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
+    _API_BASE = 'http://api.viki.io/v4/containers'
+    _APP = '100000a'
     _PER_PAGE = 25
 
     def _real_extract(self, url):
-        show_id = self._match_id(url)
-        show_page = self._download_webpage(url, show_id, 'Download show page')
+        channel_id = self._match_id(url)
 
-        title = self._og_search_title(show_page)
-        description = self._og_search_description(show_page)
+        channel = self._download_json(
+            '%s/%s.json?app=%s' % (self._API_BASE, channel_id, self._APP),
+            channel_id, 'Downloading channel JSON')
+
+        titles = channel['titles']
+        title = titles.get('en') or titles[titles.keys()[0]]
+
+        descriptions = channel['descriptions']
+        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
 
         entries = []
-        for video_type in ['episodes', 'clips']:
-            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type, self._PER_PAGE)
-            while json_url is not None:
-                show_json = self._download_json(
-                    json_url, show_id,
-                    note='Downloading %s json page #%s' %
-                         (video_type, re.search(r'[?&]page=([0-9]+)', json_url).group(1)))
-                for video in show_json['response']:
+        for video_type in ('episodes', 'clips'):
+            page_url = '%s/%s/%s.json?app=%s&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (self._API_BASE, channel_id, video_type, self._APP, self._PER_PAGE)
+            while page_url:
+                page = self._download_json(
+                    page_url, channel_id,
+                    'Downloading %s JSON page #%s'
+                    % (video_type, re.search(r'[?&]page=([0-9]+)', page_url).group(1)))
+                for video in page['response']:
                     video_id = video['id']
                     entries.append(self.url_result(
                         'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
-                json_url = show_json['pagination']['next']
+                page_url = page['pagination']['next']
 
-        return self.playlist_result(entries, show_id, title, description)
+        return self.playlist_result(entries, channel_id, title, description)

From 1a83c731bd58ed85f6f7695cee9c88d09a224bc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 01:44:05 +0600
Subject: [PATCH 0660/1515] [viki] Switch extraction to API

---
 youtube_dl/extractor/viki.py | 230 +++++++++++++++++++++--------------
 1 file changed, 142 insertions(+), 88 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fc585c299..234649ca8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,29 +1,64 @@
 from __future__ import unicode_literals
 
 import re
+import time
+import hmac
+import hashlib
 
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
-    unescapeHTML,
-    unified_strdate,
-    US_RATINGS,
-    determine_ext,
-    mimetype2ext,
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
 )
 from .common import InfoExtractor
 
 
-class VikiIE(InfoExtractor):
+class VikiBaseIE(InfoExtractor):
+    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
+    _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
+
+    _APP = '65535a'
+    _APP_VERSION = '2.2.5.1428709186'
+    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
+
+    def _prepare_call(self, path, timestamp=None):
+        path += '?' if '?' not in path else '&'
+        if not timestamp:
+            timestamp = int(time.time())
+        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        sig = hmac.new(
+            self._APP_SECRET.encode('ascii'),
+            query.encode('ascii'),
+            hashlib.sha1
+        ).hexdigest()
+        return self._API_URL_TEMPLATE % (query, sig)
+
+    def _call_api(self, path, video_id, note, timestamp=None):
+        resp = self._download_json(
+            self._prepare_call(path, timestamp), video_id, note)
+
+        error = resp.get('error')
+        if error:
+            if error == 'invalid timestamp':
+                resp = self._download_json(
+                    self._prepare_call(path, int(resp['current_timestamp'])),
+                    video_id, '%s (retry)' % note)
+                error = resp.get('error')
+            if error:
+                self._raise_error(resp['error'])
+
+        return resp
+
+    def _raise_error(self, error):
+        raise ExtractorError(
+            '%s returned error: %s' % (self.IE_NAME, error),
+            expected=True)
+
+
+class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
-
-    # iPad2
-    _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
-
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:videos|player)/(?P<id>[0-9]+v)'
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -37,115 +72,134 @@ class VikiIE(InfoExtractor):
         },
         'skip': 'Blocked in the US',
     }, {
+        # clip
         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
-        'md5': 'ca6493e6f0a6ec07da9aa8d6304b4b2c',
+        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
         'info_dict': {
             'id': '1067139v',
             'ext': 'mp4',
+            'title': "'The Avengers: Age of Ultron' Press Conference",
             'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
+            'duration': 352,
+            'timestamp': 1430380829,
             'upload_date': '20150430',
-            'title': '\'The Avengers: Age of Ultron\' Press Conference',
+            'uploader': 'Arirang TV',
+            'like_count': int,
+            'age_limit': 0,
         }
     }, {
         'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
         'info_dict': {
             'id': '1048879v',
             'ext': 'mp4',
-            'upload_date': '20140820',
-            'description': 'md5:54ff56d51bdfc7a30441ec967394e91c',
             'title': 'Ankhon Dekhi',
+            'duration': 6512,
+            'timestamp': 1408532356,
+            'upload_date': '20140820',
+            'uploader': 'Spuul',
+            'like_count': int,
+            'age_limit': 13,
         },
         'params': {
-            # requires ffmpeg
+            # m3u8 download
             'skip_download': True,
         }
+    }, {
+        # episode
+        'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
+        'md5': '190f3ef426005ba3a080a63325955bc3',
+        'info_dict': {
+            'id': '44699v',
+            'ext': 'mp4',
+            'title': 'Boys Over Flowers - Episode 1',
+            'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
+            'duration': 4155,
+            'timestamp': 1270496524,
+            'upload_date': '20100405',
+            'uploader': 'group8',
+            'like_count': int,
+            'age_limit': 13,
+        }
+    }, {
+        'url': 'http://www.viki.com/player/44699v',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        streams = self._call_api(
+            'videos/%s/streams.json' % video_id, video_id,
+            'Downloading video streams JSON')
 
-        uploader_m = re.search(
-            r'<strong>Broadcast Network: </strong>\s*([^<]*)<', webpage)
-        if uploader_m is None:
-            uploader = None
-        else:
-            uploader = uploader_m.group(1).strip()
+        formats = []
+        for format_id, stream_dict in streams.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            for protocol, format_dict in stream_dict.items():
+                if format_id == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                else:
+                    formats.append({
+                        'url': format_dict['url'],
+                        'format_id': '%s-%s' % (format_id, protocol),
+                        'height': height,
+                    })
+        self._sort_formats(formats)
 
-        rating_str = self._html_search_regex(
-            r'<strong>Rating: </strong>\s*([^<]*)<', webpage,
-            'rating information', default='').strip()
-        age_limit = US_RATINGS.get(rating_str)
+        video = self._call_api(
+            'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
-        req = compat_urllib_request.Request(
-            'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id)
-        req.add_header('User-Agent', self._USER_AGENT)
-        info_webpage = self._download_webpage(
-            req, video_id, note='Downloading info page')
-        err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
-        if err_msg:
-            if 'not available in your region' in err_msg:
-                raise ExtractorError(
-                    'Video %s is blocked from your location.' % video_id,
-                    expected=True)
-            else:
-                raise ExtractorError('Viki said: %s %s' % (err_msg, url))
-        mobj = re.search(
-            r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
-        if not mobj:
-            raise ExtractorError('Unable to find video URL')
-        video_url = unescapeHTML(mobj.group('url'))
-        video_ext = mimetype2ext(mobj.group('mime_type'))
+        title = None
+        titles = video.get('titles')
+        if titles:
+            title = titles.get('en') or titles[titles.keys()[0]]
+        if not title:
+            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
+            container_titles = video.get('container', {}).get('titles')
+            if container_titles:
+                container_title = container_titles.get('en') or container_titles[titles.keys()[0]]
+                title = '%s - %s' % (container_title, title)
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(
-                video_url, video_id, ext=video_ext)
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': video_ext,
-            }]
+        descriptions = video.get('descriptions')
+        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
 
-        upload_date_str = self._html_search_regex(
-            r'"created_at":"([^"]+)"', info_webpage, 'upload date')
-        upload_date = (
-            unified_strdate(upload_date_str)
-            if upload_date_str is not None
-            else None
-        )
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('created_at'))
+        uploader = video.get('author')
+        like_count = int_or_none(video.get('likes', {}).get('count'))
+        age_limit = parse_age_limit(video.get('rating'))
 
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, info_webpage)
+        thumbnails = []
+        for thumbnail_id, thumbnail in video.get('images', {}).items():
+            thumbnails.append({
+                'id': thumbnail_id,
+                'url': thumbnail.get('url'),
+            })
+
+        subtitles = {}
+        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
+            subtitles[subtitle_lang] = [{
+                'ext': subtitles_format,
+                'url': self._prepare_call(
+                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
+            } for subtitles_format in ('srt', 'vtt')]
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
             'description': description,
-            'thumbnail': thumbnail,
-            'age_limit': age_limit,
+            'duration': duration,
+            'timestamp': timestamp,
             'uploader': uploader,
-            'subtitles': video_subtitles,
-            'upload_date': upload_date,
+            'like_count': like_count,
+            'age_limit': age_limit,
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
         }
 
-    def _get_subtitles(self, video_id, info_webpage):
-        res = {}
-        for sturl_html in re.findall(r'<track src="([^"]+)"', info_webpage):
-            sturl = unescapeHTML(sturl_html)
-            m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl)
-            if not m:
-                continue
-            res[m.group('lang')] = [{
-                'url': compat_urlparse.urljoin('http://www.viki.com', sturl),
-                'ext': 'vtt',
-            }]
-        return res
-
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'

From ac20d95f9766aa130748aac07fa90ee5dfa566d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 01:56:02 +0600
Subject: [PATCH 0661/1515] [viki] Add support for youtube externals

---
 youtube_dl/extractor/viki.py | 70 ++++++++++++++++++++++++------------
 1 file changed, 48 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 234649ca8..68d5cac6e 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -120,6 +120,23 @@ class VikiIE(VikiBaseIE):
             'like_count': int,
             'age_limit': 13,
         }
+    }, {
+        # youtube external
+        'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
+        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
+        'info_dict': {
+            'id': '50562v',
+            'ext': 'mp4',
+            'title': 'Poor Nastya [COMPLETE] - Episode 1',
+            'description': '',
+            'duration': 607,
+            'timestamp': 1274949505,
+            'upload_date': '20101213',
+            'uploader': 'ad14065n',
+            'uploader_id': 'ad14065n',
+            'like_count': int,
+            'age_limit': 13,
+        }
     }, {
         'url': 'http://www.viki.com/player/44699v',
         'only_matching': True,
@@ -128,26 +145,6 @@ class VikiIE(VikiBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        streams = self._call_api(
-            'videos/%s/streams.json' % video_id, video_id,
-            'Downloading video streams JSON')
-
-        formats = []
-        for format_id, stream_dict in streams.items():
-            height = self._search_regex(
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
-            for protocol, format_dict in stream_dict.items():
-                if format_id == 'm3u8':
-                    formats = self._extract_m3u8_formats(
-                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
-                else:
-                    formats.append({
-                        'url': format_dict['url'],
-                        'format_id': '%s-%s' % (format_id, protocol),
-                        'height': height,
-                    })
-        self._sort_formats(formats)
-
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
@@ -186,7 +183,7 @@ class VikiIE(VikiBaseIE):
                     'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
             } for subtitles_format in ('srt', 'vtt')]
 
-        return {
+        result = {
             'id': video_id,
             'title': title,
             'description': description,
@@ -196,10 +193,39 @@ class VikiIE(VikiBaseIE):
             'like_count': like_count,
             'age_limit': age_limit,
             'thumbnails': thumbnails,
-            'formats': formats,
             'subtitles': subtitles,
         }
 
+        streams = self._call_api(
+            'videos/%s/streams.json' % video_id, video_id,
+            'Downloading video streams JSON')
+
+        if 'external' in streams:
+            result.update({
+                '_type': 'url_transparent',
+                'url': streams['external']['url'],
+            })
+            return result
+
+        formats = []
+        for format_id, stream_dict in streams.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            for protocol, format_dict in stream_dict.items():
+                if format_id == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                else:
+                    formats.append({
+                        'url': format_dict['url'],
+                        'format_id': '%s-%s' % (format_id, protocol),
+                        'height': height,
+                    })
+        self._sort_formats(formats)
+
+        result['formats'] = formats
+        return result
+
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'

From bc56355ec6bc823fe96e31688cd3123dc18ae627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:08:13 +0600
Subject: [PATCH 0662/1515] [viki:channel] Switch to API

---
 youtube_dl/extractor/viki.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 68d5cac6e..071e280fb 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -4,6 +4,7 @@ import re
 import time
 import hmac
 import hashlib
+import itertools
 
 from ..utils import (
     ExtractorError,
@@ -227,7 +228,7 @@ class VikiIE(VikiBaseIE):
         return result
 
 
-class VikiChannelIE(InfoExtractor):
+class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
     _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
@@ -247,16 +248,15 @@ class VikiChannelIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
-    _API_BASE = 'http://api.viki.io/v4/containers'
-    _APP = '100000a'
+
     _PER_PAGE = 25
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        channel = self._download_json(
-            '%s/%s.json?app=%s' % (self._API_BASE, channel_id, self._APP),
-            channel_id, 'Downloading channel JSON')
+        channel = self._call_api(
+            'containers/%s.json' % channel_id, channel_id,
+            'Downloading channel JSON')
 
         titles = channel['titles']
         title = titles.get('en') or titles[titles.keys()[0]]
@@ -266,16 +266,16 @@ class VikiChannelIE(InfoExtractor):
 
         entries = []
         for video_type in ('episodes', 'clips'):
-            page_url = '%s/%s/%s.json?app=%s&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (self._API_BASE, channel_id, video_type, self._APP, self._PER_PAGE)
-            while page_url:
-                page = self._download_json(
-                    page_url, channel_id,
-                    'Downloading %s JSON page #%s'
-                    % (video_type, re.search(r'[?&]page=([0-9]+)', page_url).group(1)))
+            for page_num in itertools.count(1):
+                page = self._call_api(
+                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
+                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
+                    'Downloading %s JSON page #%d' % (video_type, page_num))
                 for video in page['response']:
                     video_id = video['id']
                     entries.append(self.url_result(
-                        'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
-                page_url = page['pagination']['next']
+                        'http://www.viki.com/videos/%s' % video_id, 'Viki'))
+                if not page['pagination']['next']:
+                    break
 
         return self.playlist_result(entries, channel_id, title, description)

From d01924f48810db69d572bc121ab98021f04ac957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:30:04 +0600
Subject: [PATCH 0663/1515] [viki:channel] Extend matching URLs and extract
 movies

---
 youtube_dl/extractor/viki.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 071e280fb..3acb481f9 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -230,7 +230,7 @@ class VikiIE(VikiBaseIE):
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:tv|news|movies|artists)/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {
@@ -247,6 +247,15 @@ class VikiChannelIE(VikiBaseIE):
             'description': 'md5:05bf5471385aa8b21c18ad450e350525',
         },
         'playlist_count': 127,
+    }, {
+        'url': 'http://www.viki.com/news/24569c-showbiz-korea',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viki.com/artists/2141c-shinee',
+        'only_matching': True,
     }]
 
     _PER_PAGE = 25
@@ -265,7 +274,7 @@ class VikiChannelIE(VikiBaseIE):
         description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
 
         entries = []
-        for video_type in ('episodes', 'clips'):
+        for video_type in ('episodes', 'clips', 'movies'):
             for page_num in itertools.count(1):
                 page = self._call_api(
                     'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'

From 4d8ee01389c4229f14fad45f0aa7b033a2509aef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:38:43 +0600
Subject: [PATCH 0664/1515] [viki] Fix typo

---
 youtube_dl/extractor/viki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 3acb481f9..0ec8ef0ef 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -157,7 +157,7 @@ class VikiIE(VikiBaseIE):
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
             container_titles = video.get('container', {}).get('titles')
             if container_titles:
-                container_title = container_titles.get('en') or container_titles[titles.keys()[0]]
+                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                 title = '%s - %s' % (container_title, title)
 
         descriptions = video.get('descriptions')

From 4d2f42361e02bb67de7c2017c6817b46ff3b2bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 May 2015 11:42:20 +0200
Subject: [PATCH 0665/1515] [viki] remove unused import

---
 youtube_dl/extractor/viki.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0ec8ef0ef..fe7229952 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import time
 import hmac
 import hashlib

From e7752cd57853718c6875b02517613d14c4c7221d Mon Sep 17 00:00:00 2001
From: frenchy1983 <frenchy1983@zoho.com>
Date: Thu, 21 May 2015 11:47:16 +0200
Subject: [PATCH 0666/1515] [TNAFlix] Allow dot (and more) in cat_id and
 display_id

URLs with dots were raising a "UnsupportedError: Unsupported URL" error.
---
 youtube_dl/extractor/tnaflix.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index d48cbbf14..725edd3c7 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,23 +10,23 @@ from ..utils import (
 
 
 class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[\w-]+)/(?P<display_id>[\w-]+)/video(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[^/]+)/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
     _TEST = {
-        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+        'md5': '6c431ea56756497e227fb3f01a687869',
         'info_dict': {
-            'id': '553878',
-            'display_id': 'Carmella-Decesare-striptease',
+            'id': '358632',
+            'display_id': 'bunzHD-Ms.Donk',
             'ext': 'mp4',
-            'title': 'Carmella Decesare - striptease',
-            'description': '',
+            'title': 'bunzHD Ms.Donk',
+            'description': 'bubble booty ebony teen goddess Ms.Donk has a firm ass and acts like she is shy but really she is a freak in the sheets watch her 20 min XX rated vid at bunzHD.com click on the catalog link',
             'thumbnail': 're:https?://.*\.jpg$',
-            'duration': 91,
+            'duration': 394,
             'age_limit': 18,
         }
     }

From 6ad9cb224a7d9156109fe0b0100d277b954063d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 May 2015 12:02:53 +0200
Subject: [PATCH 0667/1515] [mitele] It now uses m3u8 (#5764)

It should also be possible to use Adobe HDS, but it would require more work.
---
 youtube_dl/extractor/mitele.py    | 9 +++++++--
 youtube_dl/extractor/telecinco.py | 4 ++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index d8897eb90..7091f3335 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -20,7 +20,6 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': '6a75fe9d0d3275bead0cb683c616fddb',
         'info_dict': {
             'id': '0fce117d',
             'ext': 'mp4',
@@ -29,6 +28,10 @@ class MiTeleIE(InfoExtractor):
             'display_id': 'programa-144',
             'duration': 2913,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -56,12 +59,14 @@ class MiTeleIE(InfoExtractor):
             episode,
             transform_source=strip_jsonp
         )
+        formats = self._extract_m3u8_formats(
+            token_info['tokenizedUrl'], episode, ext='mp4')
 
         return {
             'id': embed_data['videoId'],
             'display_id': episode,
             'title': info_el.find('title').text,
-            'url': token_info['tokenizedUrl'],
+            'formats': formats,
             'description': get_element_by_attribute('class', 'text', webpage),
             'thumbnail': info_el.find('thumb').text,
             'duration': parse_duration(info_el.find('duration').text),
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 251a68680..a0c744fd1 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -16,6 +16,10 @@ class TelecincoIE(MiTeleIE):
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,

From 663004ac2b001b9be03bd951d539a62cf83c58ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 22:06:25 +0600
Subject: [PATCH 0668/1515] [options] Clarify `--metadata-from-title`
 additional templates

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 22dbc3aec..dd07266b7 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -713,7 +713,7 @@ def parseOpts(overrideArguments=None):
         help='Parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
              'the parsed parameters replace existing values. '
-             'Additional templates: %(album), %(artist). '
+             'Additional templates: %(album)s, %(artist)s. '
              'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise"')
     postproc.add_option(

From 53de95da5e40aa1a465668977e507ccc914099f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 22:27:22 +0600
Subject: [PATCH 0669/1515] [viki] Extend _VALID_URLs

---
 youtube_dl/extractor/viki.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fe7229952..7f2fb1ca8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -15,6 +15,7 @@ from .common import InfoExtractor
 
 
 class VikiBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
     _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
     _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
 
@@ -58,7 +59,7 @@ class VikiBaseIE(InfoExtractor):
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:videos|player)/(?P<id>[0-9]+v)'
+    _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -229,7 +230,7 @@ class VikiIE(VikiBaseIE):
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:tv|news|movies|artists)/(?P<id>[0-9]+c)'
+    _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {

From 5cd47a5e4f54033bcf6d80908e00eff4c75a51c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 23:58:46 +0600
Subject: [PATCH 0670/1515] [videott] Fix for python 3.2

---
 youtube_dl/extractor/videott.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index ececc7ee0..591024ead 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -43,7 +43,7 @@ class VideoTtIE(InfoExtractor):
 
         formats = [
             {
-                'url': base64.b64decode(res['u']).decode('utf-8'),
+                'url': base64.b64decode(res['u'].encode('utf-8')).decode('utf-8'),
                 'ext': 'flv',
                 'format_id': res['l'],
             } for res in settings['res'] if res['u']

From 06947add03b6b619292812a771993d3365b0e7e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:03:47 +0600
Subject: [PATCH 0671/1515] [chilloutzone] Fix for python 3.2

---
 youtube_dl/extractor/chilloutzone.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dl/extractor/chilloutzone.py
index c922f6959..0206d96db 100644
--- a/youtube_dl/extractor/chilloutzone.py
+++ b/youtube_dl/extractor/chilloutzone.py
@@ -57,7 +57,7 @@ class ChilloutzoneIE(InfoExtractor):
 
         base64_video_info = self._html_search_regex(
             r'var cozVidData = "(.+?)";', webpage, 'video data')
-        decoded_video_info = base64.b64decode(base64_video_info).decode("utf-8")
+        decoded_video_info = base64.b64decode(base64_video_info.encode('utf-8')).decode('utf-8')
         video_info_dict = json.loads(decoded_video_info)
 
         # get video information from dict

From 878563c847fa5248eedbd44187536dec04643eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:06:10 +0600
Subject: [PATCH 0672/1515] [aes] Fix for python 3.2

---
 youtube_dl/aes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 07224d508..7817adcfd 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -152,7 +152,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     """
     NONCE_LENGTH_BYTES = 8
 
-    data = bytes_to_intlist(base64.b64decode(data))
+    data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
     password = bytes_to_intlist(password.encode('utf-8'))
 
     key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))

From afe8b594be53161f68189e15a65b4e9c6eba0b35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:09:15 +0600
Subject: [PATCH 0673/1515] [rtve.es:alacarta] Fix for python 3.2

---
 youtube_dl/extractor/rtve.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 849300140..82cd98ac7 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 def _decrypt_url(png):
-    encrypted_data = base64.b64decode(png)
+    encrypted_data = base64.b64decode(png.encode('utf-8'))
     text_index = encrypted_data.find(b'tEXt')
     text_chunk = encrypted_data[text_index - 4:]
     length = struct_unpack('!I', text_chunk[:4])[0]

From 43150d7ac36efda7bc60c694b8a18e1f720da04b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:10:05 +0600
Subject: [PATCH 0674/1515] [shared] Fix for python 3.2

---
 youtube_dl/extractor/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 26ced716e..9f3e944e7 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -47,7 +47,7 @@ class SharedIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'data-url="([^"]+)"', video_page, 'video URL')
         title = base64.b64decode(self._html_search_meta(
-            'full:title', webpage, 'title')).decode('utf-8')
+            'full:title', webpage, 'title').encode('utf-8')).decode('utf-8')
         filesize = int_or_none(self._html_search_meta(
             'full:size', webpage, 'file size', fatal=False))
         thumbnail = self._html_search_regex(

From 0459432d962bf358566340eed00f6c1c56b7b732 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:10:53 +0600
Subject: [PATCH 0675/1515] [shared] Fix for python 3.2

---
 youtube_dl/extractor/tutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index 4de0aac52..fad720b68 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -26,7 +26,7 @@ class TutvIE(InfoExtractor):
 
         data_content = self._download_webpage(
             'http://tu.tv/flvurl.php?codVideo=%s' % internal_id, video_id, 'Downloading video info')
-        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0]).decode('utf-8')
+        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0].encode('utf-8')).decode('utf-8')
 
         return {
             'id': internal_id,

From 77d9cb2f04462677f2a36f487c20e7a7992a0a32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:45:33 +0600
Subject: [PATCH 0676/1515] [sportbox] Fix extraction

---
 youtube_dl/extractor/sportbox.py | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 8686f9d11..2ab3489e4 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
-    parse_iso8601,
+    unified_strdate,
 )
 
 
@@ -20,11 +20,9 @@ class SportBoxIE(InfoExtractor):
             'id': '80822',
             'ext': 'mp4',
             'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-            'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+            'description': 'md5:3d72dc4a006ab6805d82f037fdc637ad',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'timestamp': 1411896237,
             'upload_date': '20140928',
-            'duration': 4846,
         },
         'params': {
             # m3u8 download
@@ -48,17 +46,13 @@ class SportBoxIE(InfoExtractor):
             r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
 
         title = self._html_search_regex(
-            r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
-        description = self._html_search_regex(
-            r'(?s)<div itemprop="description">(.+?)</div>',
-            webpage, 'description', fatal=False)
+            [r'"nodetitle"\s*:\s*"([^"]+)"', r'class="node-header_{1,2}title">([^<]+)'],
+            webpage, 'title')
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
         thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate">([^<]+)</span>',
-            webpage, 'timestamp', fatal=False))
-        duration = parse_duration(self._html_search_regex(
-            r'<meta itemprop="duration" content="PT([^"]+)">',
-            webpage, 'duration', fatal=False))
+        upload_date = unified_strdate(self._html_search_meta(
+            'dateCreated', webpage, 'upload date'))
 
         return {
             '_type': 'url_transparent',
@@ -67,8 +61,7 @@ class SportBoxIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
+            'upload_date': upload_date,
         }
 
 

From 8a278a1d7ef6134a5ac6b7dd31e3458d05f71225 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 May 2015 13:26:50 +0800
Subject: [PATCH 0677/1515] [nba] Fix duration extraction (fixes #5777)

---
 youtube_dl/extractor/nba.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 862b706bf..944096e1c 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -22,6 +22,18 @@ class NBAIE(InfoExtractor):
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
+    }, {
+        'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'info_dict': {
+            'id': '0041400301-cle-atl-recap.nba',
+            'ext': 'mp4',
+            'title': 'NBA GAME TIME | Video: Hawks vs. Cavaliers Game 1',
+            'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
+            'duration': 228,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
@@ -35,8 +47,12 @@ class NBAIE(InfoExtractor):
             self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
 
         description = self._og_search_description(webpage)
-        duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration'))
+        duration_str = self._html_search_meta(
+            'duration', webpage, 'duration', default=None)
+        if not duration_str:
+            duration_str = self._html_search_regex(
+                r'Duration:</b>\s*(\d+:\d+)', webpage, 'duration', fatal=False)
+        duration = parse_duration(duration_str)
 
         return {
             'id': shortened_video_id,

From ed5a637d62e8ede4a8cef75df4e5f341e3c667a1 Mon Sep 17 00:00:00 2001
From: frenchy1983 <frenchy1983@zoho.com>
Date: Fri, 22 May 2015 09:29:35 +0200
Subject: [PATCH 0678/1515] [TNAFlix] Restore test

See dstftw's comment in #5772
---
 youtube_dl/extractor/empflix.py | 26 ++++++++++++++------------
 youtube_dl/extractor/tnaflix.py | 32 +++++++++++++++++++-------------
 2 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 70f8efe27..0dc947c1d 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -10,16 +10,18 @@ class EMPFlixIE(TNAFlixIE):
     _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
-    _TEST = {
-        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
-        'info_dict': {
-            'id': '33051',
-            'display_id': 'Amateur-Finger-Fuck',
-            'ext': 'mp4',
-            'title': 'Amateur Finger Fuck',
-            'description': 'Amateur solo finger fucking.',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'age_limit': 18,
+    _TESTS = [
+        {
+            'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
+            'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+            'info_dict': {
+                'id': '33051',
+                'display_id': 'Amateur-Finger-Fuck',
+                'ext': 'mp4',
+                'title': 'Amateur Finger Fuck',
+                'description': 'Amateur solo finger fucking.',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'age_limit': 18,
+            }
         }
-    }
+    ]
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 725edd3c7..79496039d 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -16,20 +16,26 @@ class TNAFlixIE(InfoExtractor):
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
-    _TEST = {
-        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-        'md5': '6c431ea56756497e227fb3f01a687869',
-        'info_dict': {
-            'id': '358632',
-            'display_id': 'bunzHD-Ms.Donk',
-            'ext': 'mp4',
-            'title': 'bunzHD Ms.Donk',
-            'description': 'bubble booty ebony teen goddess Ms.Donk has a firm ass and acts like she is shy but really she is a freak in the sheets watch her 20 min XX rated vid at bunzHD.com click on the catalog link',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'duration': 394,
-            'age_limit': 18,
+    _TESTS = [
+        {
+            'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+            'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+            'info_dict': {
+                'id': '553878',
+                'display_id': 'Carmella-Decesare-striptease',
+                'ext': 'mp4',
+                'title': 'Carmella Decesare - striptease',
+                'description': '',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'duration': 91,
+                'age_limit': 18,
+            }
+        },
+        {
+            'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+            'matching_only': True,
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From ba6454761687e099f960b50cb50a9b87f4ec6d17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 11:35:09 +0200
Subject: [PATCH 0679/1515] [sportbox] Remove unused import

---
 youtube_dl/extractor/sportbox.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 2ab3489e4..86d509ae5 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    parse_duration,
     unified_strdate,
 )
 

From 79979c689713fd28e8fdf08bd71eecb6798f23d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 16:14:55 +0200
Subject: [PATCH 0680/1515] Clarify that --dump-pages encodes the pages using
 base64 (#5781)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index dd07266b7..5a2315bd9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -537,7 +537,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
-        help='Print downloaded pages to debug problems (very verbose)')
+        help='Print downloaded pages encoded using base64 to debug problems (very verbose)')
     verbosity.add_option(
         '--write-pages',
         action='store_true', dest='write_pages', default=False,

From 69e0f1b445388e4b6f45868d53780d6f8937f56e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 00:08:10 +0600
Subject: [PATCH 0681/1515] Credit @ping for viki:channel, qqmusic:toplist

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 267b8da1e..ebed7ebb3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -124,3 +124,4 @@ Mohammad Teimori Pabandi
 Roman Le Négrate
 Matthias Küch
 Julian Richen
+Ping O.

From 685c74d315a54154c5a1d9ecee8b212dbee94bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 01:01:47 +0600
Subject: [PATCH 0682/1515] [rutv] Extend embed URL (Closes #5782)

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 55604637d..d9df06861 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -104,7 +104,7 @@ class RUTVIE(InfoExtractor):
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.rutv\.ru/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From d386878af96f368ba4c2fc8bc9b078a69b79fdf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 21:25:53 +0600
Subject: [PATCH 0683/1515] [prosiebensat1] Add support for .at domain names
 (Closes #5786)

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 7cc799664..255d4abc1 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -17,7 +17,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|ran|the-voice-of-germany)\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {

From abca34cbc04693662d913e19634c06c214a237f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:04:02 +0800
Subject: [PATCH 0684/1515] [cnn] Relax _VALID_URL again (fixes #5737)

The problem is the same as test:CNN_1, so I didn't add the test case
---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 5efc5f4fe..3b1bd4033 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z]{3,5})(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',

From 9bf87ae3aaac81df3efb92fd0a3247ccb522de2a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:36:47 +0800
Subject: [PATCH 0685/1515] [nextmedia] Merge AppleDailyRealtimeNewsIE and
 AppleDailyAnimationNewsIE

---
 youtube_dl/extractor/__init__.py  |  3 +--
 youtube_dl/extractor/nextmedia.py | 37 +++++++++++++------------------
 2 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 24efb7ce5..8bb3926a0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -338,8 +338,7 @@ from .newstube import NewstubeIE
 from .nextmedia import (
     NextMediaIE,
     NextMediaActionNewsIE,
-    AppleDailyRealtimeNewsIE,
-    AppleDailyAnimationNewsIE
+    AppleDailyIE,
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index 02dba4ef6..c75ccafc7 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -89,8 +89,8 @@ class NextMediaActionNewsIE(NextMediaIE):
         return self._extract_from_nextmedia_page(news_id, url, article_page)
 
 
-class AppleDailyRealtimeNewsIE(NextMediaIE):
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+class AppleDailyIE(NextMediaIE):
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(animation|realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -99,7 +99,7 @@ class AppleDailyRealtimeNewsIE(NextMediaIE):
             'ext': 'mp4',
             'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:b23787119933404ce515c6356a8c355c',
+            'description': 'md5:2acd430e59956dc47cd7f67cb3c003f4',
             'upload_date': '20150128',
         }
     }, {
@@ -110,26 +110,10 @@ class AppleDailyRealtimeNewsIE(NextMediaIE):
             'ext': 'mp4',
             'title': '不滿被踩腳　山東兩大媽一路打下車',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:2648aaf6fc4f401f6de35a91d111aa1d',
+            'description': 'md5:175b4260c1d7c085993474217e4ab1b4',
             'upload_date': '20150128',
         }
-    }]
-
-    _URL_PATTERN = r'\{url: \'(.+)\'\}'
-
-    def _fetch_title(self, page):
-        return self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title')
-
-    def _fetch_thumbnail(self, page):
-        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
-
-    def _fetch_timestamp(self, page):
-        return None
-
-
-class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
-    _VALID_URL = 'http://www.appledaily.com.tw/animation/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
-    _TESTS = [{
+    }, {
         'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
         'md5': '03df296d95dedc2d5886debbb80cb43f',
         'info_dict': {
@@ -156,8 +140,17 @@ class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
         ]
     }]
 
+    _URL_PATTERN = r'\{url: \'(.+)\'\}'
+
     def _fetch_title(self, page):
-        return self._html_search_meta('description', page, 'news title')
+        return (self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title', default=None) or
+                self._html_search_meta('description', page, 'news title'))
+
+    def _fetch_thumbnail(self, page):
+        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
+
+    def _fetch_timestamp(self, page):
+        return None
 
     def _fetch_description(self, page):
         return self._html_search_meta('description', page, 'news description')

From 30455ce2554d00489901a398d457fac89456fe49 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:42:01 +0800
Subject: [PATCH 0686/1515] [nextmedia] Extend and reorder _VALID_URL

---
 youtube_dl/extractor/nextmedia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c75ccafc7..d1b7cff4c 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -90,7 +90,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 
 class AppleDailyIE(NextMediaIE):
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(animation|realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -138,6 +138,9 @@ class AppleDailyIE(NextMediaIE):
         'expected_warnings': [
             'video thumbnail',
         ]
+    }, {
+        'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
+        'only_matching': True,
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From 1335c3aca8f0cbddc0c521c73579eec2b9a5643c Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 24 May 2015 01:21:18 +0600
Subject: [PATCH 0687/1515] [drtv] Improve extraction (Closes #5792)

---
 youtube_dl/extractor/drtv.py | 37 +++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index f25ab319e..baa24c6d1 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,8 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor, ExtractorError
-from ..utils import parse_iso8601
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
 
 
 class DRTVIE(InfoExtractor):
@@ -60,19 +63,31 @@ class DRTVIE(InfoExtractor):
                 restricted_to_denmark = asset['RestrictedToDenmark']
                 spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                 for link in asset['Links']:
-                    target = link['Target']
                     uri = link['Uri']
+                    target = link['Target']
                     format_id = target
-                    preference = -1 if target == 'HDS' else -2
+                    preference = None
                     if spoken_subtitles:
-                        preference -= 2
+                        preference = -1
                         format_id += '-spoken-subtitles'
-                    formats.append({
-                        'url': uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43' if target == 'HDS' else uri,
-                        'format_id': format_id,
-                        'ext': link['FileFormat'],
-                        'preference': preference,
-                    })
+                    if target == 'HDS':
+                        formats.extend(self._extract_f4m_formats(
+                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
+                            video_id, preference, f4m_id=format_id))
+                    elif target == 'HLS':
+                        formats.extend(self._extract_m3u8_formats(
+                            uri, video_id, 'mp4', preference=preference,
+                            m3u8_id=format_id))
+                    else:
+                        bitrate = link.get('Bitrate')
+                        if bitrate:
+                            format_id += '-%s' % bitrate
+                        formats.append({
+                            'url': uri,
+                            'format_id': format_id,
+                            'tbr': bitrate,
+                            'ext': link.get('FileFormat'),
+                        })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
                     LANGS = {

From 71646e465348b25962a15f9a567f134514bde30a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 04:14:01 +0600
Subject: [PATCH 0688/1515] [YoutubeDL] Initialize `files_to_delete` (Closes
 #5797)

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 58b34e087..d1953c18f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1527,6 +1527,7 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
+            files_to_delete = []
             try:
                 files_to_delete, info = pp.run(info)
             except PostProcessingError as e:

From 1807ae22dd93646ea4d7ba4bd28087bf1ef4857c Mon Sep 17 00:00:00 2001
From: WassimAttar <wassim.attar@free.fr>
Date: Sun, 24 May 2015 10:37:05 +0200
Subject: [PATCH 0689/1515] chmod error

After installing youtube-dl with this method
    sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
    sudo chmod a+xr /usr/local/bin/youtube-dl
When i try to use it, i get this error
    python: can't open file '/usr/local/bin/youtube-dl': [Errno 13] Permission denied

The correct chmod is a+xr
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3d9436456..a29cccb3f 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ youtube-dl - download videos from youtube.com or other video platforms
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
     sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
-    sudo chmod a+x /usr/local/bin/youtube-dl
+    sudo chmod a+xr /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
-    sudo chmod a+x /usr/local/bin/youtube-dl
+    sudo chmod a+xr /usr/local/bin/youtube-dl
 
 Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
 

From 23905927e19280d9217ecad377ef26ea9d5793fe Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 24 May 2015 18:32:04 +0600
Subject: [PATCH 0690/1515] [README.md] Keep more idiomatic rwx order

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index a29cccb3f..e51bb5343 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ youtube-dl - download videos from youtube.com or other video platforms
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
     sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
-    sudo chmod a+xr /usr/local/bin/youtube-dl
+    sudo chmod a+rx /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
-    sudo chmod a+xr /usr/local/bin/youtube-dl
+    sudo chmod a+rx /usr/local/bin/youtube-dl
 
 Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
 

From abac15f3c6915d176c37f7aa748b8a0f03db82a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 19:08:22 +0600
Subject: [PATCH 0691/1515] [tnaflix] Do not capture `cat_id`

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 79496039d..59af9aba0 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[^/]+)/(?P<display_id>[^/]+)/video(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'

From 34fb7e46ad3fa1a04635fa4876401aac881bb39b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 19:10:03 +0600
Subject: [PATCH 0692/1515] [empflix] Relax _VALID_URL

---
 youtube_dl/extractor/empflix.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 0dc947c1d..9a5a8f4bb 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -4,7 +4,7 @@ from .tnaflix import TNAFlixIE
 
 
 class EMPFlixIE(TNAFlixIE):
-    _VALID_URL = r'^https?://www\.empflix\.com/videos/(?P<display_id>[0-9a-zA-Z-]+)-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
 
     _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
     _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
@@ -23,5 +23,9 @@ class EMPFlixIE(TNAFlixIE):
                 'thumbnail': 're:https?://.*\.jpg$',
                 'age_limit': 18,
             }
+        },
+        {
+            'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
+            'matching_only': True,
         }
     ]

From d78c834ead934e5532d2f5bc221bb11eedaef0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 20:04:13 +0600
Subject: [PATCH 0693/1515] [karrierevideos] Improve and simplify

---
 youtube_dl/extractor/karrierevideos.py | 96 +++++++++++++++++++-------
 1 file changed, 71 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index a05e8ab76..bed94bc93 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -1,50 +1,96 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    fix_xml_ampersands,
+    float_or_none,
+    xpath_with_ns,
+    xpath_text,
+)
 
 
 class KarriereVideosIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P<id>[a-z-]+)'
-    _TEST = {
+    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
         'info_dict': {
-            'id': 'altenpflegerin',
-            'ext': 'mp4',
+            'id': '32c91',
+            'ext': 'flv',
             'title': 'AltenpflegerIn',
-            'thumbnail': 're:^http://.*\.png\?v=[0-9]+',
-            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2'
+            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2',
+            'thumbnail': 're:^http://.*\.png',
         },
         'params': {
-            'skip_download': 'requires rtmpdump'
+            # rtmp download
+            'skip_download': True,
         }
-    }
+    }, {
+        # broken ampersands
+        'url': 'http://www.karrierevideos.at/orientierung/vaeterkarenz-und-neue-chancen-fuer-muetter-baby-was-nun',
+        'info_dict': {
+            'id': '5sniu',
+            'ext': 'flv',
+            'title': 'Väterkarenz und neue Chancen für Mütter - "Baby - was nun?"',
+            'description': 'md5:97092c6ad1fd7d38e9d6a5fdeb2bcc33',
+            'thumbnail': 're:^http://.*\.png',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
+        title = (self._html_search_meta('title', webpage, default=None) or
+                 self._search_regex(r'<h1 class="title">([^<]+)</h1>'))
+
+        video_id = self._search_regex(
+            r'/config/video/(.+?)\.xml', webpage, 'video id')
+        playlist = self._download_xml(
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id,
+            video_id, transform_source=fix_xml_ampersands)
+
+        NS_MAP = {
+            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
+        }
+
+        def ns(path):
+            return xpath_with_ns(path, NS_MAP)
+
+        item = playlist.find('./tracklist/item')
+        video_file = xpath_text(
+            item, ns('./jwplayer:file'), 'video url', fatal=True)
+        streamer = xpath_text(
+            item, ns('./jwplayer:streamer'), 'streamer', fatal=True)
+
+        uploader = xpath_text(
+            item, ns('./jwplayer:author'), 'uploader')
+        duration = float_or_none(
+            xpath_text(item, ns('./jwplayer:duration'), 'duration'))
+
         description = self._html_search_regex(
-            r'<div class="leadtext">\n{0,}?\s{0,}<p>(.*?)</p>',
+            r'(?s)<div class="leadtext">(.+?)</div>',
             webpage, 'description')
 
-        playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
-        playlist = self._download_xml(
-            'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist,
-            video_id)
-
-        namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
-
-        item = playlist.find('tracklist/item')
-        streamer = item.find('{%s}streamer' % namespace).text
+        thumbnail = self._html_search_meta(
+            'thumbnail', webpage, 'thumbnail')
+        if thumbnail:
+            thumbnail = compat_urlparse.urljoin(url, thumbnail)
 
         return {
             'id': video_id,
-            'title': self._html_search_meta('title', webpage),
+            'url': streamer.replace('rtmpt', 'rtmp'),
+            'play_path': 'mp4:%s' % video_file,
+            'ext': 'flv',
+            'title': title,
             'description': description,
-            'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
-            'protocol': 'rtmp',
-            'url': streamer.replace('rtmpt', 'http'),
-            'play_path': 'mp4:' + item.find('{%s}file' % namespace).text,
-            'tc_url': streamer,
-            'ext': 'mp4'
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': duration,
         }

From 63f3cab4aee5da45dc9b91a9661d5d52b5a72ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 21:09:08 +0600
Subject: [PATCH 0694/1515] [rtbf] Fix extraction (Closes #5803)

---
 youtube_dl/extractor/rtbf.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index dce64e151..5a381d9ce 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -1,10 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unescapeHTML,
+)
 
 
 class RTBFIE(InfoExtractor):
@@ -16,25 +17,24 @@ class RTBFIE(InfoExtractor):
             'id': '1921274',
             'ext': 'mp4',
             'title': 'Les Diables au coeur (épisode 2)',
-            'description': 'Football - Diables Rouges',
             'duration': 3099,
-            'timestamp': 1398456336,
-            'upload_date': '20140425',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        page = self._download_webpage('https://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
+        webpage = self._download_webpage(
+            'http://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
 
-        data = json.loads(self._html_search_regex(
-            r'<div class="js-player-embed(?: player-embed)?" data-video="([^"]+)"', page, 'data video'))['data']
+        data = self._parse_json(
+            unescapeHTML(self._search_regex(
+                r'data-video="([^"]+)"', webpage, 'data video')),
+            video_id)
 
         video_url = data.get('downloadUrl') or data.get('url')
 
-        if data['provider'].lower() == 'youtube':
+        if data.get('provider').lower() == 'youtube':
             return self.url_result(video_url, 'Youtube')
 
         return {
@@ -42,8 +42,8 @@ class RTBFIE(InfoExtractor):
             'url': video_url,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
-            'thumbnail': data['thumbnail']['large'],
+            'thumbnail': data.get('thumbnail'),
             'duration': data.get('duration') or data.get('realDuration'),
-            'timestamp': data['created'],
-            'view_count': data['viewCount'],
+            'timestamp': int_or_none(data.get('created')),
+            'view_count': int_or_none(data.get('viewCount')),
         }

From 2ad5708c43a8672da547fa279e71b20c327793d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 21:25:00 +0600
Subject: [PATCH 0695/1515] [arte:future] Switch to `search_regex` for now
 (Closes #5801)

---
 youtube_dl/extractor/arte.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 8273bd6c9..fce38248d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -195,7 +195,9 @@ class ArteTVFutureIE(ArteTVPlus7IE):
     def _real_extract(self, url):
         anchor_id, lang = self._extract_url_info(url)
         webpage = self._download_webpage(url, anchor_id)
-        row = get_element_by_id(anchor_id, webpage)
+        row = self._search_regex(
+            r'(?s)id="%s"[^>]*>.+?(<div[^>]*arte_vp_url[^>]*>)' % anchor_id,
+            webpage, 'row')
         return self._extract_from_webpage(row, anchor_id, lang)
 
 

From 04b3b3df05a26a361441754afeb7ff24d0c1f559 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 11:58:52 +0200
Subject: [PATCH 0696/1515] [youtube] Remove the nondash formats (fixes #5774)

Since we use fixed values for some fields like width and height they can be wrong, and would get picked by some formats filters.
For example for https://www.youtube.com/watch?v=EQCrhbBxsjA the biggest height is 720 and for nondash formats it's set to 1440, so -f 'bestvideo[height>=1200]+bestaudio' would incorrectly pick the nondash format, instead it should report that the requested format is not available.
---
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1f9940cf5..0301682b8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1126,12 +1126,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
                 else:
-                    # Hide the formats we found through non-DASH
+                    # Remove the formats we found through non-DASH, they
+                    # contain less info and it can be wrong, because we use
+                    # fixed values (for example the resolution). See
+                    # https://github.com/rg3/youtube-dl/issues/5774 for an
+                    # example.
                     dash_keys = set(df['format_id'] for df in dash_formats)
-                    for f in formats:
-                        if f['format_id'] in dash_keys:
-                            f['format_id'] = 'nondash-%s' % f['format_id']
-                            f['preference'] = f.get('preference', 0) - 10000
+                    formats = [f for f in formats if f['format_id'] not in dash_keys]
                     formats.extend(dash_formats)
 
         # Check for malformed aspect ratio

From 4b4e1af059c0922da9770a79a68a471277303f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 24 May 2015 18:46:29 +0200
Subject: [PATCH 0697/1515] [arte] Remove unused import

---
 youtube_dl/extractor/arte.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index fce38248d..76de24477 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
     unified_strdate,
-    get_element_by_id,
     get_element_by_attribute,
     int_or_none,
     qualities,

From d41ebe146b693011eb1020ca9cd935e7db892d0b Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 24 May 2015 23:57:47 +0600
Subject: [PATCH 0698/1515] [tenplay] Fix formats and modernize (Closes #5806)

---
 youtube_dl/extractor/tenplay.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index 466155ef8..f6694149b 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -2,6 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
 
 class TenPlayIE(InfoExtractor):
@@ -49,18 +53,23 @@ class TenPlayIE(InfoExtractor):
             if protocol == 'rtmp':
                 url = url.replace('&mp4:', '')
 
+                tbr = int_or_none(rendition.get('encodingRate'), 1000)
+
             formats.append({
-                'format_id': '_'.join(['rtmp', rendition['videoContainer'].lower(), rendition['videoCodec'].lower()]),
-                'width': rendition['frameWidth'],
-                'height': rendition['frameHeight'],
-                'tbr': rendition['encodingRate'] / 1024,
-                'filesize': rendition['size'],
+                'format_id': '_'.join(
+                    ['rtmp', rendition['videoContainer'].lower(),
+                     rendition['videoCodec'].lower(), '%sk' % tbr]),
+                'width': int_or_none(rendition['frameWidth']),
+                'height': int_or_none(rendition['frameHeight']),
+                'tbr': tbr,
+                'filesize': int_or_none(rendition['size']),
                 'protocol': protocol,
                 'ext': ext,
                 'vcodec': rendition['videoCodec'].lower(),
                 'container': rendition['videoContainer'].lower(),
                 'url': url,
             })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
@@ -74,8 +83,8 @@ class TenPlayIE(InfoExtractor):
                 'url': json['thumbnailURL']
             }],
             'thumbnail': json['videoStillURL'],
-            'duration': json['length'] / 1000,
-            'timestamp': float(json['creationDate']) / 1000,
-            'uploader': json['customFields']['production_company_distributor'] if 'production_company_distributor' in json['customFields'] else 'TENplay',
-            'view_count': json['playsTotal']
+            'duration': float_or_none(json.get('length'), 1000),
+            'timestamp': float_or_none(json.get('creationDate'), 1000),
+            'uploader': json.get('customFields', {}).get('production_company_distributor') or 'TENplay',
+            'view_count': int_or_none(json.get('playsTotal')),
         }

From 7198063d96003050eccb0ea59cc938f0388c0606 Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 15:26:59 -0500
Subject: [PATCH 0699/1515] [pinkbike] new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pinkbike.py | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/pinkbike.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..80bec39da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -394,6 +394,7 @@ from .pbs import PBSIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
+from .pinkbike import PinkbikeIE
 from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
new file mode 100644
index 000000000..4a15c1835
--- /dev/null
+++ b/youtube_dl/extractor/pinkbike.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class PinkbikeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pinkbike\.com/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.pinkbike.com/video/402811/',
+        'md5': '4814b8ca7651034cd87e3361d5c2155a',
+        'info_dict': {
+            'id': '402811',
+            'ext': 'mp4',
+            'title': 'Brandon Semenuk - RAW 100',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'location': 'Victoria, British Columbia, Canada',
+            'uploader_id': 'revelco',
+            'upload_date': '20150406',
+            'description': 'Official release: www.redbull.ca/rupertwalker',
+            'duration': '100'
+        }
+    }, {
+        'url': 'http://www.pinkbike.com/video/406629/',
+        'md5': 'c7a3e19a2bd5cde5a1cda6b2b46caa74',
+        'info_dict': {
+            'id': '406629',
+            'ext': 'mp4',
+            'title': 'Chromag: Reece Wallace in Utah',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'location': 'Whistler, British Columbia, Canada',
+            'uploader_id': 'Chromagbikes',
+            'upload_date': '20150505',
+            'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
+            'duration': '180'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        title = title[:-len(' Video - Pinkbike')]
+
+        description = self._html_search_meta('description', webpage, 'description')
+        description = description[len(title + '. '):]
+
+        uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
+
+        upload_date = self._html_search_regex(
+            r'class="fullTime"\s*title="([0-9]{4}(?:-[0-9]{2}){2})"',
+            webpage, 'upload_date')
+        upload_date = upload_date.replace('-', '')
+
+        location = self._html_search_regex(
+            r'<dt>Location</dt>\n?\s*<dd>\n?(.*?)\s*<img',
+            webpage, 'location')
+
+        formats = re.findall(
+            r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
+            webpage)
+
+        formats = [{'url': fmt[1], 'height': fmt[0]} for fmt in formats]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': self._html_search_meta('video:duration', webpage, 'duration'),
+            'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'location': location,
+            'formats': formats
+        }

From 2c935c0c7224a3332ff9f0fd83e8c074cfbe2c9d Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 16:30:03 -0500
Subject: [PATCH 0700/1515] [pinkbike] converted duration to int

---
 youtube_dl/extractor/pinkbike.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 4a15c1835..66605ddbe 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -20,7 +20,7 @@ class PinkbikeIE(InfoExtractor):
             'uploader_id': 'revelco',
             'upload_date': '20150406',
             'description': 'Official release: www.redbull.ca/rupertwalker',
-            'duration': '100'
+            'duration': 100
         }
     }, {
         'url': 'http://www.pinkbike.com/video/406629/',
@@ -34,7 +34,7 @@ class PinkbikeIE(InfoExtractor):
             'uploader_id': 'Chromagbikes',
             'upload_date': '20150505',
             'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
-            'duration': '180'
+            'duration': 180
         }
     }]
 
@@ -69,7 +69,7 @@ class PinkbikeIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': self._html_search_meta('video:duration', webpage, 'duration'),
+            'duration': int(self._html_search_meta('video:duration', webpage, 'duration')),
             'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From 680f9744c4e010ad5111c7711c58c341d5ba24dd Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 16:45:10 -0500
Subject: [PATCH 0701/1515] [pinkbike] used proper conversion methods

---
 youtube_dl/extractor/pinkbike.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 66605ddbe..45c0b1377 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -4,6 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_end,
+    remove_start
+)
 
 
 class PinkbikeIE(InfoExtractor):
@@ -43,10 +48,13 @@ class PinkbikeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        title = title[:-len(' Video - Pinkbike')]
+        title = remove_end(title, ' Video - Pinkbike')
 
         description = self._html_search_meta('description', webpage, 'description')
-        description = description[len(title + '. '):]
+        description = remove_start(description, title + '. ')
+
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration'))
 
         uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
 
@@ -63,13 +71,13 @@ class PinkbikeIE(InfoExtractor):
             r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
             webpage)
 
-        formats = [{'url': fmt[1], 'height': fmt[0]} for fmt in formats]
+        formats = [{'url': fmt[1], 'height': int_or_none(fmt[0])} for fmt in formats]
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': int(self._html_search_meta('video:duration', webpage, 'duration')),
+            'duration': duration,
             'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From b885bae6340b2aa9406501250fdebfbeea54e5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 04:53:53 +0600
Subject: [PATCH 0702/1515] Credit @misterhat for karrierevideos (#5729)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ebed7ebb3..3410e1fb9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -125,3 +125,4 @@ Roman Le Négrate
 Matthias Küch
 Julian Richen
 Ping O.
+Mister Hat

From c6bbdadd79fac001cde15e8fd118b9535427474d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 21:22:13 +0600
Subject: [PATCH 0703/1515] [odnoklassniki] Support extraction from metadata
 URL (Closes #5813)

---
 youtube_dl/extractor/odnoklassniki.py | 33 ++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index fbc521d1a..691f503f5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
     unified_strdate,
     int_or_none,
@@ -11,8 +12,9 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
+        # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
         'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
         'info_dict': {
@@ -20,11 +22,22 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'duration': 100,
-            'upload_date': '20141207',
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
-            'age_limit': 0,
+        },
+    }, {
+        # metadataUrl
+        'url': 'http://ok.ru/video/63567059965189-0',
+        'md5': '9676cf86eff5391d35dea675d224e131',
+        'info_dict': {
+            'id': '63567059965189-0',
+            'ext': 'mp4',
+            'title': 'Девушка без комплексов ...',
+            'duration': 191,
+            'uploader_id': '534380003155',
+            'uploader': 'Андрей Мещанинов',
+            'like_count': int,
         },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
@@ -41,7 +54,15 @@ class OdnoklassnikiIE(InfoExtractor):
                 r'data-attributes="([^"]+)"', webpage, 'player')),
             video_id)
 
-        metadata = self._parse_json(player['flashvars']['metadata'], video_id)
+        flashvars = player['flashvars']
+
+        metadata = flashvars.get('metadata')
+        if metadata:
+            metadata = self._parse_json(metadata, video_id)
+        else:
+            metadata = self._download_json(
+                compat_urllib_parse.unquote(flashvars['metadataUrl']),
+                video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']
         title = movie['title']
@@ -53,11 +74,11 @@ class OdnoklassnikiIE(InfoExtractor):
         uploader = author.get('name')
 
         upload_date = unified_strdate(self._html_search_meta(
-            'ya:ovs:upload_date', webpage, 'upload date'))
+            'ya:ovs:upload_date', webpage, 'upload date', default=None))
 
         age_limit = None
         adult = self._html_search_meta(
-            'ya:ovs:adult', webpage, 'age limit')
+            'ya:ovs:adult', webpage, 'age limit', default=None)
         if adult:
             age_limit = 18 if adult == 'true' else 0
 

From ba2df04b41b62d08e1fd0efaaaf104467133e9a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 21:27:43 +0600
Subject: [PATCH 0704/1515] [odnoklassniki] Make URL explicit

---
 youtube_dl/extractor/odnoklassniki.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 691f503f5..6c7149fe3 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -47,7 +47,8 @@ class OdnoklassnikiIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://ok.ru/video/%s' % video_id, video_id)
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(

From 5d0a33eebcae821ac5d1124043a5ad77a58fa291 Mon Sep 17 00:00:00 2001
From: Alexander Kirk <alexander@kirk.at>
Date: Mon, 25 May 2015 20:12:18 +0200
Subject: [PATCH 0705/1515] rtlnow is now hosted at nowtv.de

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowtv.py    |  90 ++++++++++++++++
 youtube_dl/extractor/rtlnow.py   | 174 -------------------------------
 3 files changed, 91 insertions(+), 175 deletions(-)
 create mode 100644 youtube_dl/extractor/nowtv.py
 delete mode 100644 youtube_dl/extractor/rtlnow.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..bfd07392e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,6 +355,7 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
+from .nowtv import NowTvIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
@@ -438,7 +439,6 @@ from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlNlIE
-from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
new file mode 100644
index 000000000..bf97fe7f4
--- /dev/null
+++ b/youtube_dl/extractor/nowtv.py
@@ -0,0 +1,90 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    qualities,
+    unified_strdate,
+    int_or_none,
+)
+
+class NowTvIE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'''(?x)
+                        (?:https?://)?
+                        (
+                            (?:www\.)?nowtv\.de
+                            /(rtl|rtl2|rtlnitro||superrtl|ntv|vox)(?P<path>/.*?)/player
+                        )'''
+
+    _TESTS = [
+        {
+            'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
+            'info_dict': {
+                'id': '128953',
+                'ext': 'mp4',
+                'title': 'B\u00fcro-Fall \/ Chihuahua \'Joel\'',
+                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
+                'upload_date': '2015-05-23 19:10:00',
+                'duration': '00:51:32',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'Only works from Germany',
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        info_url = 'https://api.nowtv.de/v3/movies' + mobj.group('path') + '?fields=*,format,files,breakpoints,paymentPaytypes,trailers'
+        info = self._download_json(info_url, None)
+
+        video_id = info['id']
+        title = info['title']
+        description = info['articleShort']
+        duration = info['duration']
+        upload_date = unified_strdate(info['broadcastStartDate'])
+        free = info['free']
+        station = info['format']['station']
+        thumbnail = info['format']['defaultImage169Logo']
+
+        if station == 'rtl':
+            base_url = 'http://hls.fra.rtlnow.de/hls-vod-enc/'
+        elif station == 'rtl2':
+            base_url = 'http://hls.fra.rtl2now.de/hls-vod-enc/'
+        elif station == 'vox':
+            base_url = 'http://hls.fra.voxnow.de/hls-vod-enc/'
+        elif station == 'nitro':
+            base_url = 'http://hls.fra.rtlnitronow.de/hls-vod-enc/'
+        elif station == 'ntv':
+            base_url = 'http://hls.fra.n-tvnow.de/hls-vod-enc/'
+        elif station == 'superrtl':
+            base_url = 'http://hls.fra.superrtlnow.de/hls-vod-enc/'
+
+        formats = []
+        for item in info['files']['items']:
+            if item['type'] != 'video/x-abr':
+                continue
+
+            fmt = {
+                'url': base_url + item['path'] + '.m3u8',
+                'tbr': int_or_none(item['bitrate']),
+                'ext': 'mp4',
+                'format_id': int_or_none(item['id']),
+            }
+            formats.append(fmt)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
deleted file mode 100644
index 785a8045e..000000000
--- a/youtube_dl/extractor/rtlnow.py
+++ /dev/null
@@ -1,174 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    unified_strdate,
-    int_or_none,
-)
-
-
-class RTLnowIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'''(?x)
-                        (?:https?://)?
-                        (?P<url>
-                            (?P<domain>
-                                rtl-now\.rtl\.de|
-                                rtl2now\.rtl2\.de|
-                                (?:www\.)?voxnow\.de|
-                                (?:www\.)?rtlnitronow\.de|
-                                (?:www\.)?superrtlnow\.de|
-                                (?:www\.)?n-tvnow\.de)
-                            /+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?
-                            (?:container_id|film_id)=(?P<video_id>[0-9]+)&
-                            player=1(?:&season=[0-9]+)?(?:&.*)?
-                        )'''
-
-    _TESTS = [
-        {
-            'url': 'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
-            'info_dict': {
-                'id': '90419',
-                'ext': 'flv',
-                'title': 'Ahornallee - Folge 1 - Der Einzug',
-                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
-                'upload_date': '20070416',
-                'duration': 1685,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
-        },
-        {
-            'url': 'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
-            'info_dict': {
-                'id': '69756',
-                'ext': 'flv',
-                'title': 'Ärger im Revier - Ein junger Ladendieb, ein handfester Streit u.a.',
-                'description': 'md5:3fb247005ed21a935ffc82b7dfa70cf0',
-                'thumbnail': 'http://autoimg.static-fra.de/rtl2now/219850/1500x1500/image2.jpg',
-                'upload_date': '20120519',
-                'duration': 1245,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
-        },
-        {
-            'url': 'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
-            'info_dict': {
-                'id': '13883',
-                'ext': 'flv',
-                'title': 'Voxtours - Südafrika-Reporter II',
-                'description': 'md5:de7f8d56be6fd4fed10f10f57786db00',
-                'upload_date': '20090627',
-                'duration': 1800,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://superrtlnow.de/medicopter-117/angst.php?film_id=99205&player=1',
-            'info_dict': {
-                'id': '99205',
-                'ext': 'flv',
-                'title': 'Medicopter 117 - Angst!',
-                'description': 're:^Im Therapiezentrum \'Sonnalm\' kommen durch eine Unachtsamkeit die für die B.handlung mit Phobikern gehaltenen Voglespinnen frei\. Eine Ausreißerin',
-                'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
-                'upload_date': '20080928',
-                'duration': 2691,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://rtl-now.rtl.de/der-bachelor/folge-4.php?film_id=188729&player=1&season=5',
-            'info_dict': {
-                'id': '188729',
-                'ext': 'flv',
-                'upload_date': '20150204',
-                'description': 'md5:5e1ce23095e61a79c166d134b683cecc',
-                'title': 'Der Bachelor - Folge 4',
-            }
-        }, {
-            'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
-            'only_matching': True,
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_page_url = 'http://%s/' % mobj.group('domain')
-        video_id = mobj.group('video_id')
-
-        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
-
-        mobj = re.search(r'(?s)<div style="margin-left: 20px; font-size: 13px;">(.*?)<div id="playerteaser">', webpage)
-        if mobj:
-            raise ExtractorError(clean_html(mobj.group(1)), expected=True)
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
-
-        upload_date = unified_strdate(self._html_search_meta('uploadDate', webpage, 'upload date'))
-
-        mobj = re.search(r'<meta itemprop="duration" content="PT(?P<seconds>\d+)S" />', webpage)
-        duration = int(mobj.group('seconds')) if mobj else None
-
-        playerdata_url = self._html_search_regex(
-            r"'playerdata': '(?P<playerdata_url>[^']+)'", webpage, 'playerdata_url')
-
-        playerdata = self._download_xml(playerdata_url, video_id, 'Downloading player data XML')
-
-        videoinfo = playerdata.find('./playlist/videoinfo')
-
-        formats = []
-        for filename in videoinfo.findall('filename'):
-            mobj = re.search(r'(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>.+)', filename.text)
-            if mobj:
-                fmt = {
-                    'url': mobj.group('url'),
-                    'play_path': 'mp4:' + mobj.group('play_path'),
-                    'page_url': video_page_url,
-                    'player_url': video_page_url + 'includes/vodplayer.swf',
-                }
-            else:
-                mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
-                if mobj:
-                    fmt = {
-                        'url': 'rtmpe://fms.rtl.de/' + mobj.group('hoster'),
-                        'play_path': 'mp4:' + mobj.group('play_path'),
-                        'page_url': url,
-                        'player_url': video_page_url + 'includes/vodplayer.swf',
-                    }
-                else:
-                    fmt = {
-                        'url': filename.text,
-                    }
-            fmt.update({
-                'width': int_or_none(filename.get('width')),
-                'height': int_or_none(filename.get('height')),
-                'vbr': int_or_none(filename.get('bitrate')),
-                'ext': 'flv',
-            })
-            formats.append(fmt)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'duration': duration,
-            'formats': formats,
-        }

From 42833b44b5e3810a2875dfb130aefbf5db057c1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:32:43 +0800
Subject: [PATCH 0706/1515] [tf1] Extend _VALID_URL (fixes #5819)

---
 youtube_dl/extractor/tf1.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 025d0877c..656410528 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
-    _TESTS = {
+    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou|www\.tf1)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
             'id': '10635995',
@@ -32,7 +32,10 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From db3ca36403930063b7df3b228a3f297bf278b43c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:37:15 +0800
Subject: [PATCH 0707/1515] [facebook] Move the title extraction warning below
 (fixes #5820)

---
 youtube_dl/extractor/facebook.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 937b28fcc..e8d682716 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -50,7 +50,10 @@ class FacebookIE(InfoExtractor):
             'id': '274175099429670',
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
-        }
+        },
+        'expected_warnings': [
+            'title'
+        ]
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -150,11 +153,11 @@ class FacebookIE(InfoExtractor):
 
         video_title = self._html_search_regex(
             r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
-            fatal=False)
+            default=None)
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
-                webpage, 'alternative title', default=None)
+                webpage, 'alternative title', fatal=False)
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id

From efec4358b9b8ed5726c1f7d7939e8bce49f9100c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:54:41 +0800
Subject: [PATCH 0708/1515] [cinemassacre] Support an alternative form of
 screenwavemedia URL

fixes #5821
---
 youtube_dl/extractor/cinemassacre.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index cf0a7551b..c949a4814 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -60,6 +60,17 @@ class CinemassacreIE(InfoExtractor):
                 'uploader_id': 'Cinemassacre',
                 'title': 'AVGN: McKids',
             }
+        },
+        {
+            'url': 'http://cinemassacre.com/2015/05/25/mario-kart-64-nintendo-64-james-mike-mondays/',
+            'md5': '1376908e49572389e7b06251a53cdd08',
+            'info_dict': {
+                'id': 'Cinemassacre-555779690c440',
+                'ext': 'mp4',
+                'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
+                'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
+                'upload_date': '20150525',
+            }
         }
     ]
 
@@ -72,7 +83,7 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             [
-                r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+                r'src="(http://(?:player2\.screenwavemedia\.com|player\.screenwavemedia\.com/play)/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
                 r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
             webpage, 'player data URL', default=None)

From ff305edd645d6f4307faa9307dea91694a1d217d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:43:00 +0300
Subject: [PATCH 0709/1515] [sockshare] Remove extractor

Haywire since last October.
---
 youtube_dl/extractor/sockshare.py | 83 -------------------------------
 1 file changed, 83 deletions(-)
 delete mode 100644 youtube_dl/extractor/sockshare.py

diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
deleted file mode 100644
index b5fa6f1da..000000000
--- a/youtube_dl/extractor/sockshare.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-)
-
-from .common import InfoExtractor
-
-
-class SockshareIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sockshare\.com/file/(?P<id>[0-9A-Za-z]+)'
-    _FILE_DELETED_REGEX = r'This file doesn\'t exist, or has been removed\.</div>'
-    _TEST = {
-        'url': 'http://www.sockshare.com/file/437BE28B89D799D7',
-        'md5': '9d0bf1cfb6dbeaa8d562f6c97506c5bd',
-        'info_dict': {
-            'id': '437BE28B89D799D7',
-            'title': 'big_buck_bunny_720p_surround.avi',
-            'ext': 'avi',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://sockshare.com/file/%s' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        confirm_hash = self._html_search_regex(r'''(?x)<input\s+
-            type="hidden"\s+
-            value="([^"]*)"\s+
-            name="hash"
-            ''', webpage, 'hash')
-
-        fields = {
-            "hash": confirm_hash.encode('utf-8'),
-            "confirm": "Continue as Free User"
-        }
-
-        post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
-        # Apparently, this header is required for confirmation to work.
-        req.add_header('Host', 'www.sockshare.com')
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        webpage = self._download_webpage(
-            req, video_id, 'Downloading video page')
-
-        video_url = self._html_search_regex(
-            r'<a href="([^"]*)".+class="download_file_link"',
-            webpage, 'file url')
-        video_url = "http://www.sockshare.com" + video_url
-        title = self._html_search_regex((
-            r'<h1>(.+)<strong>',
-            r'var name = "([^"]+)";'),
-            webpage, 'title', default=None)
-        thumbnail = self._html_search_regex(
-            r'<img\s+src="([^"]*)".+?name="bg"',
-            webpage, 'thumbnail', default=None)
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'ext': determine_ext(title),
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 35a4f24a378419d41ea9d9fbcc9790fa457e83d2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:44:46 +0300
Subject: [PATCH 0710/1515] [firedrive] Remove extractor (Closes #3870)

Haywire since last October.
---
 youtube_dl/extractor/firedrive.py | 80 -------------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/firedrive.py

diff --git a/youtube_dl/extractor/firedrive.py b/youtube_dl/extractor/firedrive.py
deleted file mode 100644
index 3191116d9..000000000
--- a/youtube_dl/extractor/firedrive.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-)
-
-
-class FiredriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?firedrive\.com/' + \
-                 '(?:file|embed)/(?P<id>[0-9a-zA-Z]+)'
-    _FILE_DELETED_REGEX = r'<div class="removed_file_image">'
-
-    _TESTS = [{
-        'url': 'https://www.firedrive.com/file/FEB892FA160EBD01',
-        'md5': 'd5d4252f80ebeab4dc2d5ceaed1b7970',
-        'info_dict': {
-            'id': 'FEB892FA160EBD01',
-            'ext': 'flv',
-            'title': 'bbb_theora_486kbit.flv',
-            'thumbnail': 're:^http://.*\.jpg$',
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://firedrive.com/file/%s' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', webpage))
-
-        post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        # Apparently, this header is required for confirmation to work.
-        req.add_header('Host', 'www.firedrive.com')
-
-        webpage = self._download_webpage(req, video_id,
-                                         'Downloading video page')
-
-        title = self._search_regex(r'class="external_title_left">(.+)</div>',
-                                   webpage, 'title')
-        thumbnail = self._search_regex(r'image:\s?"(//[^\"]+)', webpage,
-                                       'thumbnail', fatal=False)
-        if thumbnail is not None:
-            thumbnail = 'http:' + thumbnail
-
-        ext = self._search_regex(r'type:\s?\'([^\']+)\',',
-                                 webpage, 'extension', fatal=False)
-        video_url = self._search_regex(
-            r'file:\s?loadURL\(\'(http[^\']+)\'\),', webpage, 'file url')
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'ext': ext,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 544a8693b7d9321d776987a5104889056955daa2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:53:14 +0300
Subject: [PATCH 0711/1515] Remove Firedrive and Sockshare imports

Oops
---
 youtube_dl/extractor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..80c9cb107 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -149,7 +149,6 @@ from .extremetube import ExtremeTubeIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
-from .firedrive import FiredriveIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
@@ -480,7 +479,6 @@ from .smotri import (
     SmotriBroadcastIE,
 )
 from .snotr import SnotrIE
-from .sockshare import SockshareIE
 from .sohu import SohuIE
 from .soundcloud import (
     SoundcloudIE,

From 7d65242dc3c0c3306b775f0663d325ba55b62379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 May 2015 22:12:26 +0600
Subject: [PATCH 0712/1515] [dailymotion:user] Process user home as user
 (Closes #5823)

---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index db10b8d00..d8f9eb13f 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -225,7 +225,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:old/)?user/(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',
@@ -239,7 +239,8 @@ class DailymotionUserIE(DailymotionPlaylistIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
-        webpage = self._download_webpage(url, user)
+        webpage = self._download_webpage(
+            'https://www.dailymotion.com/user/%s' % user, user)
         full_user = unescapeHTML(self._html_search_regex(
             r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
             webpage, 'user'))

From 5406af92bc3f88b9fe4f26fe20bdaaf0b4968c32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 May 2015 22:16:47 +0600
Subject: [PATCH 0713/1515] [dailymotion:user] Fix _VALID_URL

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index d8f9eb13f..70aa4333c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -225,7 +225,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 7a372b64dfa69d5b2cfd1514b89e8fc0ab7e5874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 01:41:00 +0600
Subject: [PATCH 0714/1515] [pornhub] Do not modify aes key string (Closes
 #5824)

---
 youtube_dl/extractor/pornhub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 0c8b731cf..daa284ea2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -71,7 +71,8 @@ class PornHubIE(InfoExtractor):
 
         video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
+            password = compat_urllib_parse.unquote_plus(
+                self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 
         formats = []

From 1434184c577953ff6fe558ccc6751697791f4076 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 01:42:53 +0600
Subject: [PATCH 0715/1515] [spankwire] Do not modify aes key string

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index b936202f6..06d6e6640 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -71,7 +71,7 @@ class SpankwireIE(InfoExtractor):
             compat_urllib_parse.unquote,
             re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
-            password = self._html_search_regex(
+            password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',
                 webpage, 'password').replace('+', ' ')
             video_urls = list(map(

From b535170b218131afd1165776e611691479627ce8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 04:14:24 +0800
Subject: [PATCH 0716/1515] [bilibili] Skip assertion if HQ videos not
 available

---
 youtube_dl/extractor/bilibili.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 7ca835e31..a8bea2c10 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -80,9 +80,11 @@ class BiliBiliIE(InfoExtractor):
             note='Downloading HQ video info',
             fatal=False,
         )
-        hq_durls = hq_doc.findall('./durl') if hq_doc is not False else itertools.repeat(None)
-
-        assert len(lq_durls) == len(hq_durls)
+        if hq_doc is not False:
+            hq_durls = hq_doc.findall('./durl')
+            assert len(lq_durls) == len(hq_durls)
+        else:
+            hq_durls = itertools.repeat(None)
 
         i = 1
         for lq_durl, hq_durl in zip(lq_durls, hq_durls):

From 6d00a2dcd110f12a0aa110f5479df76613792fbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 04:23:21 +0800
Subject: [PATCH 0717/1515] [bilibili] Catch API call failures

JSON are returned in a failed API call
---
 youtube_dl/extractor/bilibili.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index a8bea2c10..2103ed73a 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 
 import re
 import itertools
+import json
+import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
@@ -67,11 +69,19 @@ class BiliBiliIE(InfoExtractor):
 
         entries = []
 
-        lq_doc = self._download_xml(
+        lq_page = self._download_webpage(
             'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
             video_id,
             note='Downloading LQ video info'
         )
+        try:
+            err_info = json.loads(lq_page)
+            raise ExtractorError(
+                'BiliBili said: ' + err_info['error_text'], expected=True)
+        except ValueError:
+            pass
+
+        lq_doc = ET.fromstring(lq_page)
         lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(

From c23848b3c5244f8ef1501adfd04a32111b12d7ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:20:29 +0800
Subject: [PATCH 0718/1515] [naver] Enhanced error detection

---
 youtube_dl/extractor/naver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index c10405f04..fdd825784 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -35,7 +35,7 @@ class NaverIE(InfoExtractor):
                          webpage)
         if m_id is None:
             m_error = re.search(
-                r'(?s)<div class="nation_error">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                r'(?s)<div class="(?:nation_error|nation_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
                 webpage)
             if m_error:
                 raise ExtractorError(clean_html(m_error.group('msg')), expected=True)

From f8d5e1cfb5d9a8c946a966452d9b86c45182a952 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:44:08 +0800
Subject: [PATCH 0719/1515] [naver] Fix video url (fixes #5809)

RTMP urls in test:naver does not work. Need more investigation.
---
 youtube_dl/extractor/naver.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index fdd825784..925967753 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -16,7 +17,7 @@ from ..utils import (
 class NaverIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tvcast.naver.com/v/81652',
         'info_dict': {
             'id': '81652',
@@ -25,7 +26,18 @@ class NaverIE(InfoExtractor):
             'description': '합격불변의 법칙 메가스터디 | 메가스터디 수학 김상희 선생님이 9월 모의고사 수학A형 16번에서 20번까지 해설강의를 공개합니다.',
             'upload_date': '20130903',
         },
-    }
+    }, {
+        'url': 'http://tvcast.naver.com/v/395837',
+        'md5': '638ed4c12012c458fefcddfd01f173cd',
+        'info_dict': {
+            'id': '395837',
+            'ext': 'mp4',
+            'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
+            'description': 'md5:5bf200dcbf4b66eb1b350d1eb9c753f7',
+            'upload_date': '20150519',
+        },
+        'skip': 'Georestricted',
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -58,14 +70,18 @@ class NaverIE(InfoExtractor):
         formats = []
         for format_el in urls.findall('EncodingOptions/EncodingOption'):
             domain = format_el.find('Domain').text
+            uri = format_el.find('uri').text
             f = {
-                'url': domain + format_el.find('uri').text,
+                'url': compat_urlparse.urljoin(domain, uri),
                 'ext': 'mp4',
                 'width': int(format_el.find('width').text),
                 'height': int(format_el.find('height').text),
             }
             if domain.startswith('rtmp'):
+                # urlparse does not support custom schemes
+                # https://bugs.python.org/issue18828
                 f.update({
+                    'url': domain + uri,
                     'ext': 'flv',
                     'rtmp_protocol': '1',  # rtmpt
                 })

From f9f3e3df9a9fff1b00c7184234c4f607ea3cec81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:51:18 +0800
Subject: [PATCH 0720/1515] [teamcoco] Use determine_ext to determine the video
 type

Some videos does not contain a 'type' field (#5798)
---
 youtube_dl/extractor/teamcoco.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 56be52638..b2a4b1fc0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     qualities,
+    determine_ext,
 )
 from ..compat import compat_ord
 
@@ -108,7 +109,7 @@ class TeamcocoIE(InfoExtractor):
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
-            if filed['type'] == 'hls':
+            if determine_ext(filed['url']) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     filed['url'], video_id, ext='mp4'))
             else:

From f0bfaa2d7d9563975f1f6effa75d28dcdb1c23ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 15:23:34 +0800
Subject: [PATCH 0721/1515] [nrk] Update subtitles test

Subtitle conversion routine is removed, so the subtitles are TTML now. See
1c7e2e64f6328024711d5fa999d4498396f4cb5c
---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 891ee620b..c4e3adb67 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -266,7 +266,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['no']))
-        self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
+        self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
 
 
 class TestRaiSubtitles(BaseTestSubtitles):

From bf24c3d01798fad0a8344a642eb5d46231fd78c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 21:25:07 +0600
Subject: [PATCH 0722/1515] [facebook] Improve title regex (Closes #5816)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index e8d682716..82dc27bc6 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -152,7 +152,7 @@ class FacebookIE(InfoExtractor):
             raise ExtractorError('Cannot find video formats')
 
         video_title = self._html_search_regex(
-            r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)
         if not video_title:
             video_title = self._html_search_regex(

From d90b3854ca9e8602f440cc9439e1cba240192286 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 00:37:00 +0800
Subject: [PATCH 0723/1515] [porn91] Add new extractor for 91porn.com

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/porn91.py   | 62 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/porn91.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 80c9cb107..d20ad286d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -400,6 +400,7 @@ from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
+from .porn91 import Porn91IE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
new file mode 100644
index 000000000..af06af2b7
--- /dev/null
+++ b/youtube_dl/extractor/porn91.py
@@ -0,0 +1,62 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from ..compat import compat_urllib_parse
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class Porn91IE(InfoExtractor):
+    IE_NAME = '91porn'
+    _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
+
+    _TEST = {
+            'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
+            'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+            'info_dict': {
+                'id': '7e42283b4f5ab36da134',
+                'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+                'ext': 'mp4'
+            }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
+        self._set_cookie('91porn.com', 'language', 'cn_CN')
+        webpage = self._download_webpage(url, video_id, "get HTML content")
+        title = re.search(
+            r'<div id="viewvideo-title">(.+?)</div>',
+            webpage,
+            re.DOTALL)
+        assert title
+        title = title.group(1).replace('\n', '')
+
+        # get real url
+        n1 = re.search(r'so.addVariable\(\'file\',\'(\d+)\'', webpage)
+        n2 = re.search(r'so.addVariable\(\'seccode\',\'(.+?)\'', webpage)
+        n3 = re.search(r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage)
+        if not (n1 and n2 and n3):
+            raise ExtractorError("You are Blocked by Server.")
+
+        url_params = compat_urllib_parse.urlencode({
+            'VID': n1.group(1),
+            'mp4': '1',
+            'seccode': n2.group(1),
+            'max_vid': n3.group(1),
+        })
+        t_url = 'http://91porn.com/getfile.php?' + url_params
+        info_cn = self._download_webpage(t_url, video_id, "get real video_url")
+        video_url = re.search(r'file=(http.+?)&', info_cn).group(1)
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+        }
+
+        return info

From b25b645d5106e5b2bf33c640813fe744b63f4730 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 23:20:32 +0600
Subject: [PATCH 0724/1515] [nowtv] Improve and simplify

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowtv.py    | 197 ++++++++++++++++++++++---------
 2 files changed, 139 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bfd07392e..17248ccea 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,7 +355,7 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
-from .nowtv import NowTvIE
+from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index bf97fe7f4..5c91acec6 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,90 +1,169 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    qualities,
-    unified_strdate,
     int_or_none,
+    parse_iso8601,
+    parse_duration,
+    remove_start,
 )
 
-class NowTvIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'''(?x)
-                        (?:https?://)?
-                        (
-                            (?:www\.)?nowtv\.de
-                            /(rtl|rtl2|rtlnitro||superrtl|ntv|vox)(?P<path>/.*?)/player
-                        )'''
 
-    _TESTS = [
-        {
-            'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
-            'info_dict': {
-                'id': '128953',
-                'ext': 'mp4',
-                'title': 'B\u00fcro-Fall \/ Chihuahua \'Joel\'',
-                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
-                'upload_date': '2015-05-23 19:10:00',
-                'duration': '00:51:32',
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
+class NowTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?P<station>rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+
+    _TESTS = [{
+        # rtl
+        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/player',
+        'info_dict': {
+            'id': '203519',
+            'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
+            'ext': 'mp4',
+            'title': 'Die neuen Bauern und eine Hochzeit',
+            'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432580700,
+            'upload_date': '20150525',
+            'duration': 2786,
         },
-    ]
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # rtl2
+        'url': 'http://www.nowtv.de/rtl2/berlin-tag-nacht/berlin-tag-nacht-folge-934/player',
+        'info_dict': {
+            'id': '203481',
+            'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
+            'ext': 'mp4',
+            'title': 'Berlin - Tag & Nacht (Folge 934)',
+            'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432666800,
+            'upload_date': '20150526',
+            'duration': 2641,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # superrtl
+        'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',
+        'info_dict': {
+            'id': '99205',
+            'display_id': 'medicopter-117/angst',
+            'ext': 'mp4',
+            'title': 'Angst!',
+            'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1222632900,
+            'upload_date': '20080928',
+            'duration': 3025,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # ntv
+        'url': 'http://www.nowtv.de/ntv/ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch/player',
+        'info_dict': {
+            'id': '203521',
+            'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
+            'ext': 'mp4',
+            'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
+            'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432751700,
+            'upload_date': '20150527',
+            'duration': 1083,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # vox
+        'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
+        'info_dict': {
+            'id': '128953',
+            'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
+            'ext': 'mp4',
+            'title': "Büro-Fall / Chihuahua 'Joel'",
+            'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432408200,
+            'upload_date': '20150523',
+            'duration': 3092,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        info_url = 'https://api.nowtv.de/v3/movies' + mobj.group('path') + '?fields=*,format,files,breakpoints,paymentPaytypes,trailers'
-        info = self._download_json(info_url, None)
+        display_id = mobj.group('id')
+        station = mobj.group('station')
 
-        video_id = info['id']
-        title = info['title']
-        description = info['articleShort']
-        duration = info['duration']
-        upload_date = unified_strdate(info['broadcastStartDate'])
-        free = info['free']
-        station = info['format']['station']
-        thumbnail = info['format']['defaultImage169Logo']
+        info = self._download_json(
+            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files,breakpoints,paymentPaytypes,trailers,pictures' % display_id,
+            display_id)
 
-        if station == 'rtl':
-            base_url = 'http://hls.fra.rtlnow.de/hls-vod-enc/'
-        elif station == 'rtl2':
-            base_url = 'http://hls.fra.rtl2now.de/hls-vod-enc/'
-        elif station == 'vox':
-            base_url = 'http://hls.fra.voxnow.de/hls-vod-enc/'
-        elif station == 'nitro':
-            base_url = 'http://hls.fra.rtlnitronow.de/hls-vod-enc/'
-        elif station == 'ntv':
-            base_url = 'http://hls.fra.n-tvnow.de/hls-vod-enc/'
-        elif station == 'superrtl':
-            base_url = 'http://hls.fra.superrtlnow.de/hls-vod-enc/'
+        video_id = compat_str(info['id'])
+
+        if info.get('geoblocked'):
+            raise ExtractorError(
+                'Video %s is not available from your location due to geo restriction' % video_id,
+                expected=True)
+
+        f = info.get('format', {})
+        station = f.get('station') or station
+
+        STATIONS = {
+            'rtl': 'rtlnow',
+            'rtl2': 'rtl2now',
+            'vox': 'voxnow',
+            'nitro': 'rtlnitronow',
+            'ntv': 'n-tvnow',
+            'superrtl': 'superrtlnow'
+        }
 
         formats = []
         for item in info['files']['items']:
-            if item['type'] != 'video/x-abr':
-                continue
-
-            fmt = {
-                'url': base_url + item['path'] + '.m3u8',
-                'tbr': int_or_none(item['bitrate']),
+            item_path = remove_start(item['path'], '/')
+            tbr = int_or_none(item['bitrate'])
+            m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)
+            m3u8_url = m3u8_url.replace('now/', 'now/videos/')
+            formats.append({
+                'url': m3u8_url,
+                'format_id': '%s-%sk' % (item['id'], tbr),
                 'ext': 'mp4',
-                'format_id': int_or_none(item['id']),
-            }
-            formats.append(fmt)
+                'tbr': tbr,
+            })
         self._sort_formats(formats)
 
+        title = info['title']
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
         }

From 703d78bbf5edf73f60447ac273c0d303d28cc340 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 01:37:24 +0800
Subject: [PATCH 0725/1515] [porn91] change re to _search_regex

---
 youtube_dl/extractor/porn91.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index af06af2b7..f3a97df64 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
@@ -29,30 +28,32 @@ class Porn91IE(InfoExtractor):
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
-        title = re.search(
-            r'<div id="viewvideo-title">(.+?)</div>',
-            webpage,
-            re.DOTALL)
+        title = self._search_regex(
+            r'<div id="viewvideo-title">(?P<title>.+?)</div>',
+            webpage, 'title', flags=re.DOTALL)
         assert title
-        title = title.group(1).replace('\n', '')
+        title = title.replace('\n', '')
 
         # get real url
-        n1 = re.search(r'so.addVariable\(\'file\',\'(\d+)\'', webpage)
-        n2 = re.search(r'so.addVariable\(\'seccode\',\'(.+?)\'', webpage)
-        n3 = re.search(r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage)
+        n1 = self._search_regex(
+            r'so.addVariable\(\'file\',\'(?P<n1>\d+)\'', webpage, 'n1')
+        n2 = self._search_regex(
+            r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
+        n3 = self._search_regex(
+            r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
         if not (n1 and n2 and n3):
             raise ExtractorError("You are Blocked by Server.")
-
         url_params = compat_urllib_parse.urlencode({
-            'VID': n1.group(1),
+            'VID': n1,
             'mp4': '1',
-            'seccode': n2.group(1),
-            'max_vid': n3.group(1),
+            'seccode': n2,
+            'max_vid': n3,
         })
         t_url = 'http://91porn.com/getfile.php?' + url_params
         info_cn = self._download_webpage(t_url, video_id, "get real video_url")
-        video_url = re.search(r'file=(http.+?)&', info_cn).group(1)
+        video_url = self._search_regex(r'file=(?P<url>http.+?)&', info_cn, 'url')
 
+        # construct info
         info = {
             'id': video_id,
             'title': title,

From 9b254aa177d58b7b4c4f44dce8c38fa7978c7df6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 23:41:43 +0600
Subject: [PATCH 0726/1515] [nowtv] Add non-free video check

---
 youtube_dl/extractor/nowtv.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 5c91acec6..295168432 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -120,10 +120,15 @@ class NowTVIE(InfoExtractor):
 
         video_id = compat_str(info['id'])
 
-        if info.get('geoblocked'):
-            raise ExtractorError(
-                'Video %s is not available from your location due to geo restriction' % video_id,
-                expected=True)
+        files = info['files']
+        if not files:
+            if info.get('geoblocked', False):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
 
         f = info.get('format', {})
         station = f.get('station') or station
@@ -138,7 +143,7 @@ class NowTVIE(InfoExtractor):
         }
 
         formats = []
-        for item in info['files']['items']:
+        for item in files['items']:
             item_path = remove_start(item['path'], '/')
             tbr = int_or_none(item['bitrate'])
             m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)

From ff4a1279f2d40fdba3287d4e7949bd8caa89eb04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 May 2015 01:15:04 +0600
Subject: [PATCH 0727/1515] [nowtv] Do not request unnecessary metadata

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 295168432..d39bbde99 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -115,7 +115,7 @@ class NowTVIE(InfoExtractor):
         station = mobj.group('station')
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files,breakpoints,paymentPaytypes,trailers,pictures' % display_id,
+            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files' % display_id,
             display_id)
 
         video_id = compat_str(info['id'])

From 9e0b5791281c68e5773555688928184064396011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 May 2015 01:26:14 +0600
Subject: [PATCH 0728/1515] [nowtv] Add test for rtlnitro

---
 youtube_dl/extractor/nowtv.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index d39bbde99..173e46cd8 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -53,6 +53,24 @@ class NowTVIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        # rtlnitro
+        'url': 'http://www.nowtv.de/rtlnitro/alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00/player',
+        'info_dict': {
+            'id': '165780',
+            'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
+            'ext': 'mp4',
+            'title': 'Hals- und Beinbruch',
+            'description': 'md5:b50d248efffe244e6f56737f0911ca57',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432415400,
+            'upload_date': '20150523',
+            'duration': 2742,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # superrtl
         'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',

From f9355dc989362f31d8e21ccb8fa765546a2360f2 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 17:00:09 +0800
Subject: [PATCH 0729/1515] [youku] update youku

---
 youtube_dl/extractor/youku.py | 248 ++++++++++++++++++++++------------
 1 file changed, 162 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 97b98bbe8..8d86c3f45 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,123 +1,199 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-import math
-import random
 import re
-import time
+import base64
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
-
+from ..utils import ExtractorError
 
 class YoukuIE(InfoExtractor):
+    IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
         (?:
             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
             youku:)
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
+
     _TEST = {
-        'url': 'http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html',
-        'md5': 'ffe3f2e435663dc2d1eea34faeff5b5b',
-        'params': {
-            'test': False
-        },
-        'info_dict': {
-            'id': 'XNDgyMDQ2NTQw_part00',
-            'ext': 'flv',
-            'title': 'youtube-dl test video "\'/\\ä↭𝕐'
-        }
+            'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
+            'md5': '5f3af4192eabacc4501508d54a8cabd7',
+            'info_dict': {
+                'id': 'XMTc1ODE5Njcy',
+                'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
+                'ext': 'flv'
+            }
     }
 
-    def _gen_sid(self):
-        nowTime = int(time.time() * 1000)
-        random1 = random.randint(1000, 1998)
-        random2 = random.randint(1000, 9999)
+    def construct_video_urls(self, data1, data2):
+        # get sid, token
+        def yk_t(s1, s2):
+            ls = list(range(256))
+            t = 0
+            for i in range(256):
+                t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
+                ls[i], ls[t] = ls[t], ls[i]
+            s, x, y = '', 0, 0
+            for i in range(len(s2)):
+                y = (y + 1) % 256
+                x = (x + ls[y]) % 256
+                ls[x], ls[y] = ls[y], ls[x]
+                s += chr((s2[i] ^ ls[(ls[x]+ls[y]) % 256]))
+            return s
 
-        return "%d%d%d" % (nowTime, random1, random2)
+        sid, token = yk_t(
+            'becaf9be', base64.b64decode(bytes(data2['ep'], 'ascii'))
+        ).split('_')
 
-    def _get_file_ID_mix_string(self, seed):
-        mixed = []
-        source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
-        seed = float(seed)
-        for i in range(len(source)):
-            seed = (seed * 211 + 30031) % 65536
-            index = math.floor(seed / 65536 * len(source))
-            mixed.append(source[int(index)])
-            source.remove(source[int(index)])
-        # return ''.join(mixed)
-        return mixed
+        # get oip
+        oip = data2['ip']
 
-    def _get_file_id(self, fileId, seed):
-        mixed = self._get_file_ID_mix_string(seed)
-        ids = fileId.split('*')
-        realId = []
-        for ch in ids:
-            if ch:
-                realId.append(mixed[int(ch)])
-        return ''.join(realId)
+        # get fileid
+        string_ls = list(
+            'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
+        shuffled_string_ls = []
+        seed = data1['seed']
+        N = len(string_ls)
+        for ii in range(N):
+            seed = (seed * 0xd3 + 0x754f) % 0x10000
+            idx = seed * len(string_ls) // 0x10000
+            shuffled_string_ls.append(string_ls[idx])
+            del string_ls[idx]
+
+        fileid_dict = {}
+        for format in data1['streamtypes']:
+            streamfileid = [
+                int(i) for i in data1['streamfileids'][format].strip('*').split('*')]
+            fileid = ''.join(
+                [shuffled_string_ls[i] for i in streamfileid])
+            fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
+
+        def get_fileid(format, n):
+            fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            return fileid
+
+        # get ep
+        def generate_ep(format, n):
+            fileid = get_fileid(format, n)
+            ep_t = yk_t(
+                'bf7e5f01',
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii'))
+            ep = base64.b64encode(bytes(ep_t, 'latin')).decode()
+            ep = ep.replace('+', '%2B')
+            ep = ep.replace('/', '%2F')
+            ep = ep.replace('=', '%2D')
+            return ep
+
+        # generate video_urls
+        video_urls_dict = {}
+        for format in data1['streamtypes']:
+            video_urls = []
+            for dt in data1['segs'][format]:
+                n = str(int(dt['no']))
+                video_url = \
+                    'http://k.youku.com/player/getFlvPath/' + \
+                    'sid/' + sid + \
+                    '_' + str(int(n)+1).zfill(2) + \
+                    '/st/' + self.parse_ext_l(format) + \
+                    '/fileid/' + get_fileid(format, n)  + '?' + \
+                    'K=' + str(dt['k']) + \
+                    '&hd=' + self.get_hd(format) + \
+                    '&myp=0' + \
+                    '&ts=' + str(dt['seconds']) + \
+                    '&ypp=0&ctype=12&ev=1' + \
+                    '&token=' + str(token) + \
+                    '&oip=' + str(oip) + \
+                    '&ep=' + generate_ep(format, n)
+                video_urls.append(video_url)
+            video_urls_dict[format] = video_urls
+
+        return video_urls_dict
+
+    def get_hd(self, fm):
+        hd_id_dict = {
+            'flv': '0',
+            'mp4': '1',
+            'hd2': '2',
+            'hd3': '3',
+            '3gp': '0',
+            '3gphd': '1'
+        }
+        return hd_id_dict[fm]
+
+    def parse_ext_l(self, fm):
+        ext_dict = {
+            'flv': 'flv',
+            'mp4': 'mp4',
+            'hd2': 'flv',
+            'hd3': 'flv',
+            '3gp': 'flv',
+            '3gphd': 'mp4',
+        }
+        return ext_dict[fm]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
+        # request basic data
+        data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        data2_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id
 
-        config = self._download_json(info_url, video_id)
+        raw_data1 = self._download_json(data1_url, video_id)
+        raw_data2 = self._download_json(data2_url, video_id)
+        data1 = raw_data1['data'][0]
+        data2 = raw_data2['data'][0]
 
-        error_code = config['data'][0].get('error_code')
+        error_code = data1.get('error_code')
         if error_code:
             # -8 means blocked outside China.
-            error = config['data'][0].get('error')  # Chinese and English, separated by newline.
-            raise ExtractorError(error or 'Server reported error %i' % error_code,
-                                 expected=True)
+            # Chinese and English, separated by newline.
+            error = data1.get('error')
+            raise ExtractorError(
+                error or 'Server reported error %i' %
+                error_code,
+                expected=True)
 
-        video_title = config['data'][0]['title']
-        seed = config['data'][0]['seed']
+        title = data1['title']
 
-        format = self._downloader.params.get('format', None)
-        supported_format = list(config['data'][0]['streamfileids'].keys())
+        # generate video_urls_dict
+        video_urls_dict = self.construct_video_urls(data1, data2)
 
-        # TODO proper format selection
-        if format is None or format == 'best':
-            if 'hd2' in supported_format:
-                format = 'hd2'
-            else:
-                format = 'flv'
-            ext = 'flv'
-        elif format == 'worst':
-            format = 'mp4'
-            ext = 'mp4'
-        else:
-            format = 'flv'
-            ext = 'flv'
+        # construct info
+        entries = []
+        for fm in data1['streamtypes']:
+            #formats = []
+            video_urls = video_urls_dict[fm]
+            for i in range(len(video_urls)):
+                if len(entries) < i+1:
+                    entries.append({'formats': []})
+                entries[i]['formats'].append(
+                    {
+                        'url': video_urls[i],
+                        'format_id': fm,
+                        'ext': self.parse_ext_l(fm),
+                        'filesize': int(data1['segs'][fm][i]['size'])
+                    }
+                )
 
-        fileid = config['data'][0]['streamfileids'][format]
-        keys = [s['k'] for s in config['data'][0]['segs'][format]]
-        # segs is usually a dictionary, but an empty *list* if an error occured.
-
-        files_info = []
-        sid = self._gen_sid()
-        fileid = self._get_file_id(fileid, seed)
-
-        # column 8,9 of fileid represent the segment number
-        # fileid[7:9] should be changed
-        for index, key in enumerate(keys):
-            temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
-            download_url = 'http://k.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
+        for i in range(len(entries)):
+            entries[i].update(
+                {
+                    'id': '_part%d' % (i+1),
+                    'title': title,
+                }
+            )
 
+        if len(entries) > 1:
             info = {
-                'id': '%s_part%02d' % (video_id, index),
-                'url': download_url,
-                'uploader': None,
-                'upload_date': None,
-                'title': video_title,
-                'ext': ext,
+                '_type': 'multi_video',
+                'id': video_id,
+                'title': title,
+                'entries': entries,
             }
-            files_info.append(info)
+        else:
+            info = entries[0]
+            info['id'] = video_id
 
-        return files_info
+        return info

From ca45246627f5a67a7c82cd40a11e5c4ff5f68871 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 21:04:58 +0800
Subject: [PATCH 0730/1515] [youku] compatible for python > 3.3 or > 2.7

---
 youtube_dl/extractor/youku.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 8d86c3f45..7a07c8a5f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import sys
+pyvs = sys.version_info[0]
 import re
 import base64
 
@@ -34,16 +36,23 @@ class YoukuIE(InfoExtractor):
             for i in range(256):
                 t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s, x, y = '', 0, 0
+            s = '' if pyvs == 3 else b''
+            x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
-                s += chr((s2[i] ^ ls[(ls[x]+ls[y]) % 256]))
+                if isinstance(s2[i], int):
+                    s += chr(s2[i] ^ ls[(ls[x]+ls[y]) % 256])
+                else:
+                    s += chr(ord(s2[i]) ^ ls[(ls[x]+ls[y]) % 256])
             return s
 
         sid, token = yk_t(
-            'becaf9be', base64.b64decode(bytes(data2['ep'], 'ascii'))
+            'becaf9be',
+            base64.b64decode(bytes(data2['ep'], 'ascii')) \
+                if pyvs == 3 \
+                else base64.b64decode(data2['ep'])
         ).split('_')
 
         # get oip
@@ -78,8 +87,15 @@ class YoukuIE(InfoExtractor):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
                 'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii'))
-            ep = base64.b64encode(bytes(ep_t, 'latin')).decode()
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
+                if pyvs == 3 \
+                else ('%s_%s_%s' % (sid, fileid, token))
+            )
+            ep = base64.b64encode(
+                bytes(ep_t, 'latin') \
+                if pyvs == 3 \
+                else ep_t
+            ).decode()
             ep = ep.replace('+', '%2B')
             ep = ep.replace('/', '%2F')
             ep = ep.replace('=', '%2D')

From 806598b94dec1268566ae71d671116060f7971d6 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 08:21:24 +0800
Subject: [PATCH 0731/1515] [porn91] the one that _search_regex returns not
 needs to be checked

---
 youtube_dl/extractor/porn91.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index f3a97df64..b62eec92d 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -31,7 +31,6 @@ class Porn91IE(InfoExtractor):
         title = self._search_regex(
             r'<div id="viewvideo-title">(?P<title>.+?)</div>',
             webpage, 'title', flags=re.DOTALL)
-        assert title
         title = title.replace('\n', '')
 
         # get real url
@@ -41,8 +40,6 @@ class Porn91IE(InfoExtractor):
             r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
         n3 = self._search_regex(
             r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
-        if not (n1 and n2 and n3):
-            raise ExtractorError("You are Blocked by Server.")
         url_params = compat_urllib_parse.urlencode({
             'VID': n1,
             'mp4': '1',

From 1498940b10a3f43490c05045ebe7a517267a2bff Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 10:13:09 +0800
Subject: [PATCH 0732/1515] [youku] compare bytes and str for compatible; use
 compat_urllib_parse for making video_url

---
 youtube_dl/extractor/youku.py | 38 +++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 7a07c8a5f..063f2e10e 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,14 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import sys
-pyvs = sys.version_info[0]
 import re
 import base64
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
+from ..compat import compat_urllib_parse
+
+bytes_is_str = (bytes == str)  # for compatible
+
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
@@ -36,7 +38,7 @@ class YoukuIE(InfoExtractor):
             for i in range(256):
                 t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s = '' if pyvs == 3 else b''
+            s = '' if not bytes_is_str else b''
             x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
@@ -51,7 +53,7 @@ class YoukuIE(InfoExtractor):
         sid, token = yk_t(
             'becaf9be',
             base64.b64decode(bytes(data2['ep'], 'ascii')) \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else base64.b64decode(data2['ep'])
         ).split('_')
 
@@ -88,17 +90,14 @@ class YoukuIE(InfoExtractor):
             ep_t = yk_t(
                 'bf7e5f01',
                 bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else ('%s_%s_%s' % (sid, fileid, token))
             )
             ep = base64.b64encode(
                 bytes(ep_t, 'latin') \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else ep_t
             ).decode()
-            ep = ep.replace('+', '%2B')
-            ep = ep.replace('/', '%2F')
-            ep = ep.replace('=', '%2D')
             return ep
 
         # generate video_urls
@@ -107,20 +106,25 @@ class YoukuIE(InfoExtractor):
             video_urls = []
             for dt in data1['segs'][format]:
                 n = str(int(dt['no']))
+                param = {
+                    'K': dt['k'],
+                    'hd': self.get_hd(format),
+                    'myp': 0,
+                    'ts': dt['seconds'],
+                    'ypp': 0,
+                    'ctype': 12,
+                    'ev': 1,
+                    'token': token,
+                    'oip': oip,
+                    'ep': generate_ep(format, n)
+                }
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
                     '_' + str(int(n)+1).zfill(2) + \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n)  + '?' + \
-                    'K=' + str(dt['k']) + \
-                    '&hd=' + self.get_hd(format) + \
-                    '&myp=0' + \
-                    '&ts=' + str(dt['seconds']) + \
-                    '&ypp=0&ctype=12&ev=1' + \
-                    '&token=' + str(token) + \
-                    '&oip=' + str(oip) + \
-                    '&ep=' + generate_ep(format, n)
+                    compat_urllib_parse.urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
 

From 84e1e036c2cb7311cdea14763bec3322403a8d54 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 May 2015 12:44:31 +0800
Subject: [PATCH 0733/1515] [senate] Extend _VALID_URL (fixes #5836)

---
 youtube_dl/extractor/senateisvp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index d3b8a1be4..9c53704ea 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -48,7 +48,7 @@ class SenateISVPIE(InfoExtractor):
         ["arch", "", "http://ussenate-f.akamaihd.net/"]
     ]
     _IE_NAME = 'senate.gov'
-    _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
+    _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
@@ -72,12 +72,16 @@ class SenateISVPIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Integrated Senate Video Player'
         }
+    }, {
+        # From http://www.c-span.org/video/?96791-1
+        'url': 'http://www.senate.gov/isvp?type=live&comm=banking&filename=banking012715',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _search_iframe_url(webpage):
         mobj = re.search(
-            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/\?[^'\"]+)['\"]",
+            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/?\?[^'\"]+)['\"]",
             webpage)
         if mobj:
             return mobj.group('url')

From eb6cb9fbe934fe99a35af22065cf91063d416c12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 29 May 2015 07:52:17 +0200
Subject: [PATCH 0734/1515] release 2015.05.29

---
 README.md              | 4 ++--
 docs/supportedsites.md | 9 ++++-----
 youtube_dl/version.py  | 2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index e51bb5343..f3d83c89f 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress                    Do not print progress bar
     --console-title                  Display progress in console titlebar
     -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages to debug problems (very verbose)
+    --dump-pages                     Print downloaded pages encoded using base64 to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging
@@ -220,7 +220,7 @@ which means you can modify it, redistribute it or use it however you like.
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
-                                     parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
+                                     parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a4879bd9a..a421ae62b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -26,8 +26,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
- - **AppleDailyAnimationNews**
- - **AppleDailyRealtimeNews**
+ - **AppleDaily**
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -152,7 +151,6 @@
  - **fc2**
  - **fernsehkritik.tv**
  - **fernsehkritik.tv:postecke**
- - **Firedrive**
  - **Firstpost**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
@@ -230,6 +228,7 @@
  - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
  - **Karaoketv**
+ - **KarriereVideos**
  - **keek**
  - **KeezMovies**
  - **KhanAcademy**
@@ -322,6 +321,7 @@
  - **NosVideo**
  - **novamov**: NovaMov
  - **Nowness**
+ - **NowTV**
  - **nowvideo**: NowVideo
  - **npo.nl**
  - **npo.nl:live**
@@ -393,7 +393,6 @@
  - **Rte**
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
- - **RTLnow**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
@@ -431,7 +430,6 @@
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
- - **Sockshare**
  - **Sohu**
  - **soundcloud**
  - **soundcloud:playlist**
@@ -564,6 +562,7 @@
  - **vier:videos**
  - **Viewster**
  - **viki**
+ - **viki:channel**
  - **vimeo**
  - **vimeo:album**
  - **vimeo:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b33385153..653710131 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.20'
+__version__ = '2015.05.29'

From d6aa68ce75ef4f4d27fbf7103edfda1f92ba70b2 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 29 May 2015 12:47:20 +0200
Subject: [PATCH 0735/1515] [postprocessor/embedthumbnail] embed mp4 too (fixes
 #5840)

---
 youtube_dl/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 8f825f785..774494efd 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -49,7 +49,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        elif info['ext'] == 'm4a':
+        elif info['ext'] in ['m4a', 'mp4']:
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
@@ -82,6 +82,6 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 os.remove(encodeFilename(filename))
                 os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         else:
-            raise EmbedThumbnailPPError('Only mp3 and m4a are supported for thumbnail embedding for now.')
+            raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
 
         return [], info

From 605ec701b7b4cd120a9acb33bfcc4306719b59b4 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 23:32:04 +0800
Subject: [PATCH 0736/1515] [iqiyi] Add new extractor for iqiyi.com

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/iqiyi.py    | 214 +++++++++++++++++++++++++++++++
 2 files changed, 215 insertions(+)
 create mode 100644 youtube_dl/extractor/iqiyi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 80c9cb107..85c1b1a3a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -229,6 +229,7 @@ from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
+from .iqiyi import IqiyiIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
new file mode 100644
index 000000000..d96d13225
--- /dev/null
+++ b/youtube_dl/extractor/iqiyi.py
@@ -0,0 +1,214 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_chr,
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
+    compat_str,
+)
+
+from ..utils import ExtractorError
+
+import re
+import time
+import json
+import uuid
+import math
+import random
+import zlib
+import hashlib
+
+class IqiyiIE(InfoExtractor):
+    IE_NAME = 'iqiyi'
+
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
+
+    _TEST = {
+            'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
+            'md5': '260f0f59686e65e886995d0ba791ab83',
+            'info_dict': {
+                'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+                'title': '美国德州空中惊现奇异云团 酷似UFO',
+                'ext': 'f4v'
+            }
+    }
+
+    def construct_video_urls(self, data, video_id, _uuid):
+        def do_xor(x, y):
+            a = y % 3
+            if a == 1:
+                return x ^ 121
+            if a == 2:
+                return x ^ 72
+            return x ^ 103
+
+        def get_encode_code(l):
+            a = 0
+            b = l.split('-')
+            c = len(b)
+            s = ''
+            for i in range(c - 1, -1, -1):
+                a = do_xor(int(b[c-i-1], 16), i)
+                s += chr(a)
+            return s[::-1]
+
+        def get_path_key(x):
+            mg = ')(*&^flash@#$%a'
+            tm = self._download_json(
+                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
+            t = str(int(math.floor(int(tm)/(600.0))))
+            return hashlib.md5(
+                 (t+mg+x).encode('utf8')).hexdigest()
+
+        video_urls_dict = {}
+        for i in data['vp']['tkl'][0]['vs']:
+            if 0 < int(i['bid']) <= 10:
+                format_id = self.get_format(i['bid'])
+
+            video_urls_info = i['fs']
+            if not i['fs'][0]['l'].startswith('/'):
+                t = get_encode_code(i['fs'][0]['l'])
+                if t.endswith('mp4'):
+                    video_urls_info = i['flvs']
+
+            video_urls = []
+            for ii in video_urls_info:
+                vl = ii['l']
+                if not vl.startswith('/'):
+                    vl = get_encode_code(vl)
+                key = get_path_key(
+                    vl.split('/')[-1].split('.')[0])
+                filesize = ii['b']
+                base_url = data['vp']['du'].split('/')
+                base_url.insert(-1, key)
+                base_url = '/'.join(base_url)
+                param = {
+                    'su': _uuid,
+                    'qyid': uuid.uuid4().hex,
+                    'client': '',
+                    'z': '',
+                    'bt': '',
+                    'ct': '',
+                    'tn': str(int(time.time()))
+                }
+                api_video_url = base_url + vl + '?' + \
+                    compat_urllib_parse.urlencode(param)
+                js = self._download_json(api_video_url, video_id)
+                video_url = js['l']
+                video_urls.append(
+                    (video_url, filesize))
+
+            video_urls_dict[format_id] = video_urls
+        return video_urls_dict
+
+    def get_format(self, bid):
+        bid_dict = {
+            '1': 'standard',
+            '2': 'high',
+            '3': 'super',
+            '4': 'suprt-high',
+            '5': 'fullhd',
+            '10': '4k'
+        }
+        return bid_dict[str(bid)]
+
+    def get_raw_data(self, tvid, video_id, enc_key, _uuid):
+        tm = str(int(time.time()))
+        param = {
+            'key': 'fvip',
+            'src': hashlib.md5(b'youtube-dl').hexdigest(),
+            'tvId': tvid,
+            'vid': video_id,
+            'vinfo': 1,
+            'tm': tm,
+            'enc': hashlib.md5(
+                (enc_key + tm + tvid).encode('utf8')).hexdigest(),
+            'qyid': _uuid,
+            'tn': random.random(),
+            'um': 0,
+            'authkey': hashlib.md5(
+                (tm + tvid).encode('utf8')).hexdigest()
+        }
+
+        api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
+            compat_urllib_parse.urlencode(param)
+        raw_data = self._download_json(api_url, video_id)
+        return raw_data
+
+    def get_enc_key(self, swf_url, video_id):
+        req = self._request_webpage(
+            swf_url, video_id, note='download swf content')
+        cn = req.read()
+        cn = zlib.decompress(cn[8:])
+        pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
+        enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
+        return enc_key
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(
+            url, 'temp_id', note='download video page')
+        tvid = self._search_regex(
+            r'tvId ?= ?(\'|\")(?P<tvid>\d+)', webpage, 'tvid', flags=re.I, group='tvid')
+        video_id = self._search_regex(
+            r'videoId ?= ?(\'|\")(?P<video_id>[a-z\d]+)',
+            webpage, 'video_id', flags=re.I, group='video_id')
+        swf_url = self._search_regex(
+            r'(?P<swf>http://.+?MainPlayer.+?\.swf)', webpage, 'swf')
+        _uuid = uuid.uuid4().hex
+
+        enc_key = self.get_enc_key(swf_url, video_id)
+
+        raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
+        assert raw_data['code'] == 'A000000'
+        if not raw_data['data']['vp']['tkl']:
+            raise ExtractorError('No support iQiqy VIP video')
+
+        data = raw_data['data']
+
+        title = data['vi']['vn']
+
+        # generate video_urls_dict
+        video_urls_dict = self.construct_video_urls(data, video_id, _uuid)
+
+        # construct info
+        entries = []
+        for format_id in video_urls_dict:
+            video_urls = video_urls_dict[format_id]
+            for i, video_url_info in enumerate(video_urls):
+                if len(entries) < i+1:
+                    entries.append({'formats': []})
+                entries[i]['formats'].append(
+                    {
+                        'url': video_url_info[0],
+                        'filesize': video_url_info[-1],
+                        'format_id': format_id,
+                    }
+                )
+
+        for i in range(len(entries)):
+            entries[i].update(
+                {
+                    'id': '_part%d' % (i+1),
+                    'title': title,
+                }
+            )
+
+        if len(entries) > 1:
+            info = {
+                '_type': 'multi_video',
+                'id': video_id,
+                'title': title,
+                'entries': entries,
+            }
+        else:
+            info = entries[0]
+            info['id'] = video_id
+            info['title'] = title
+
+        return info

From 08f7db20c16743a2bd3040eb7dac11d675011eef Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Sat, 30 May 2015 10:03:32 +0800
Subject: [PATCH 0737/1515] [youku] change format_id

---
 youtube_dl/extractor/youku.py | 37 +++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 063f2e10e..aed6b960a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -132,26 +132,37 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv': '0',
-            'mp4': '1',
-            'hd2': '2',
-            'hd3': '3',
-            '3gp': '0',
-            '3gphd': '1'
+            'flv'   : '0',
+            'mp4'   : '1',
+            'hd2'   : '2',
+            'hd3'   : '3',
+            '3gp'   : '0',
+            '3gphd' : '1'
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
-            'flv': 'flv',
-            'mp4': 'mp4',
-            'hd2': 'flv',
-            'hd3': 'flv',
-            '3gp': 'flv',
-            '3gphd': 'mp4',
+            'flv'   : 'flv',
+            'mp4'   : 'mp4',
+            'hd2'   : 'flv',
+            'hd3'   : 'flv',
+            '3gp'   : 'flv',
+            '3gphd' : 'mp4'
         }
         return ext_dict[fm]
 
+    def get_format_name(self, fm):
+        _dict = {
+            '3gp'   : 'h6',
+            '3gphd' : 'h5',
+            'flv'   : 'h4',
+            'mp4'   : 'h3',
+            'hd2'   : 'h2',
+            'hd3'   : 'h1'
+        }
+        return _dict[fm]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -191,7 +202,7 @@ class YoukuIE(InfoExtractor):
                 entries[i]['formats'].append(
                     {
                         'url': video_urls[i],
-                        'format_id': fm,
+                        'format_id': self.get_format_name(fm),
                         'ext': self.parse_ext_l(fm),
                         'filesize': int(data1['segs'][fm][i]['size'])
                     }

From 670861bd206ab4063baeb6b80d06a054ce4e1d62 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Sat, 30 May 2015 10:37:54 +0800
Subject: [PATCH 0738/1515] [iqiyi] Do not request for unneeded formats

---
 youtube_dl/extractor/iqiyi.py | 72 ++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d96d13225..747f3f902 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -4,20 +4,12 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_chr,
-    compat_parse_qs,
-    compat_urllib_parse,
-    compat_urllib_request,
-    compat_urlparse,
-    compat_str,
-)
+from ..compat import compat_urllib_parse
 
 from ..utils import ExtractorError
 
 import re
 import time
-import json
 import uuid
 import math
 import random
@@ -31,15 +23,15 @@ class IqiyiIE(InfoExtractor):
 
     _TEST = {
             'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-            'md5': '260f0f59686e65e886995d0ba791ab83',
+            'md5': '2cb594dc2781e6c941a110d8f358118b',
             'info_dict': {
                 'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
                 'title': '美国德州空中惊现奇异云团 酷似UFO',
-                'ext': 'f4v'
+                'ext': 'f4v',
             }
     }
 
-    def construct_video_urls(self, data, video_id, _uuid):
+    def construct_video_urls(self, data, video_id, _uuid, bid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -66,10 +58,21 @@ class IqiyiIE(InfoExtractor):
             return hashlib.md5(
                  (t+mg+x).encode('utf8')).hexdigest()
 
+        # get accept format
+        # getting all format will spend minutes for a big video.
+        if bid == 'best':
+            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs'] \
+                   if 0 < int(i['bid']) <= 10]
+            bid = str(max(bids))
+
         video_urls_dict = {}
         for i in data['vp']['tkl'][0]['vs']:
             if 0 < int(i['bid']) <= 10:
                 format_id = self.get_format(i['bid'])
+            else:
+                continue
+
+            video_urls = []
 
             video_urls_info = i['fs']
             if not i['fs'][0]['l'].startswith('/'):
@@ -77,7 +80,12 @@ class IqiyiIE(InfoExtractor):
                 if t.endswith('mp4'):
                     video_urls_info = i['flvs']
 
-            video_urls = []
+            if int(i['bid']) != int(bid):  # ignore missing match format
+                video_urls.extend(
+                    [('http://example.com/v.flv', ii['b']) for ii in video_urls_info])
+                video_urls_dict[format_id] = video_urls
+                continue
+
             for ii in video_urls_info:
                 vl = ii['l']
                 if not vl.startswith('/'):
@@ -108,15 +116,27 @@ class IqiyiIE(InfoExtractor):
         return video_urls_dict
 
     def get_format(self, bid):
-        bid_dict = {
-            '1': 'standard',
-            '2': 'high',
-            '3': 'super',
-            '4': 'suprt-high',
-            '5': 'fullhd',
-            '10': '4k'
+        _dict = {
+            '1'  : 'h6',
+            '2'  : 'h5',
+            '3'  : 'h4',
+            '4'  : 'h3',
+            '5'  : 'h2',
+            '10' : 'h1'
         }
-        return bid_dict[str(bid)]
+        return _dict.get(str(bid), None)
+
+    def get_bid(self, format_id):
+        _dict = {
+            'h6'   : '1',
+            'h5'   : '2',
+            'h4'   : '3',
+            'h3'   : '4',
+            'h2'   : '5',
+            'h1'   : '10',
+            'best' : 'best'
+        }
+        return _dict.get(format_id, None)
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))
@@ -173,8 +193,14 @@ class IqiyiIE(InfoExtractor):
 
         title = data['vi']['vn']
 
+        format = self._downloader.params.get('format', None)
+        bid = self.get_bid(format) if format else 'best'
+        if not bid:
+            raise ExtractorError('Can\'t get format.')
+
         # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data, video_id, _uuid)
+        video_urls_dict = self.construct_video_urls(
+            data, video_id, _uuid, bid)
 
         # construct info
         entries = []
@@ -188,10 +214,12 @@ class IqiyiIE(InfoExtractor):
                         'url': video_url_info[0],
                         'filesize': video_url_info[-1],
                         'format_id': format_id,
+                        'preference': int(self.get_bid(format_id))
                     }
                 )
 
         for i in range(len(entries)):
+            self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
                     'id': '_part%d' % (i+1),

From fafec39d4177f4873bb2393749a46873c4ffda4a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 May 2015 13:23:09 +0800
Subject: [PATCH 0739/1515] [spiegeltv] Changed RTMP server (fixes #5788 and
 fixes #5843)

Thanks to @brickleroux for finding out the problem
---
 youtube_dl/extractor/spiegeltv.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 98cf92d89..359722ad6 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -51,9 +51,9 @@ class SpiegeltvIE(InfoExtractor):
         is_wide = media_json['is_wide']
 
         server_json = self._download_json(
-            'http://www.spiegel.tv/streaming_servers/', video_id,
-            note='Downloading server information')
-        server = server_json[0]['endpoint']
+            'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
+            video_id, note='Downloading server information')
+        server = server_json['streamingserver'][0]['endpoint']
 
         thumbnails = []
         for image in media_json['images']:
@@ -76,5 +76,6 @@ class SpiegeltvIE(InfoExtractor):
             'ext': 'm4v',
             'description': description,
             'duration': duration,
-            'thumbnails': thumbnails
+            'thumbnails': thumbnails,
+            'rtmp_live': True,
         }

From 6ebdfe43e439239df051f6071a23c51705c150cf Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 30 May 2015 09:30:14 +0200
Subject: [PATCH 0740/1515] [tube8] fix extractor (fixes #5846)

---
 youtube_dl/extractor/tube8.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index d73ad3762..6ca8840b0 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -47,7 +47,7 @@ class Tube8IE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
-            r'var flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
+            r'flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
 
         video_url = flashvars['video_url']
         if flashvars.get('encrypted') is True:
@@ -58,19 +58,19 @@ class Tube8IE(InfoExtractor):
         thumbnail = flashvars.get('image_url')
 
         title = self._html_search_regex(
-            r'videotitle\s*=\s*"([^"]+)', webpage, 'title')
+            r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'>Description:</strong>(.+?)<', webpage, 'description', fatal=False)
+            r'>Description:</strong>\s*(.+?)\s*<', webpage, 'description', fatal=False)
         uploader = self._html_search_regex(
-            r'<strong class="video-username">(?:<a href="[^"]+">)?([^<]+)(?:</a>)?</strong>',
+            r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
 
         like_count = int_or_none(self._html_search_regex(
-            r"rupVar\s*=\s*'(\d+)'", webpage, 'like count', fatal=False))
+            r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
         dislike_count = int_or_none(self._html_search_regex(
-            r"rdownVar\s*=\s*'(\d+)'", webpage, 'dislike count', fatal=False))
+            r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
         view_count = self._html_search_regex(
-            r'<strong>Views: </strong>([\d,\.]+)</li>', webpage, 'view count', fatal=False)
+            r'<strong>Views: </strong>([\d,\.]+)\s*</li>', webpage, 'view count', fatal=False)
         if view_count:
             view_count = str_to_int(view_count)
         comment_count = self._html_search_regex(

From 0385d642232ba4e8b455d0c4eb95c7985f22f276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:12:58 +0600
Subject: [PATCH 0741/1515] [crunchyroll] Extract subtitles extraction routine

---
 youtube_dl/extractor/crunchyroll.py | 30 +++++++++++++----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 1c77df47e..4ac537a6d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -76,8 +76,8 @@ class CrunchyrollIE(InfoExtractor):
         self._login()
 
     def _decrypt_subtitles(self, data, iv, id):
-        data = bytes_to_intlist(data)
-        iv = bytes_to_intlist(iv)
+        data = bytes_to_intlist(base64.b64decode(data))
+        iv = bytes_to_intlist(base64.b64decode(iv))
         id = int(id)
 
         def obfuscate_key_aux(count, modulo, start):
@@ -179,6 +179,16 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         return output
 
+    def _extract_subtitles(self, subtitle):
+        sub_root = xml.etree.ElementTree.fromstring(subtitle)
+        return [{
+            'ext': 'srt',
+            'data': self._convert_subtitles_to_srt(sub_root),
+        }, {
+            'ext': 'ass',
+            'data': self._convert_subtitles_to_ass(sub_root),
+        }]
+
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
@@ -190,25 +200,11 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
             if not id or not iv or not data:
                 continue
-            id = int(id)
-            iv = base64.b64decode(iv)
-            data = base64.b64decode(data)
-
             subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            subtitles[lang_code] = [
-                {
-                    'ext': 'srt',
-                    'data': self._convert_subtitles_to_srt(sub_root),
-                },
-                {
-                    'ext': 'ass',
-                    'data': self._convert_subtitles_to_ass(sub_root),
-                },
-            ]
+            subtitles[lang_code] = self._extract_subtitles(subtitle)
         return subtitles
 
     def _real_extract(self, url):

From b2cf6543b21bbe0954c45b35b1402eaca5187c0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:30:04 +0600
Subject: [PATCH 0742/1515] [soompi] Improve and simplify

---
 youtube_dl/extractor/soompi.py | 146 ++++++++++++++++++---------------
 1 file changed, 81 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
index 4726872dc..5da66ca9e 100644
--- a/youtube_dl/extractor/soompi.py
+++ b/youtube_dl/extractor/soompi.py
@@ -2,17 +2,31 @@
 from __future__ import unicode_literals
 
 import re
-import json
-import base64
-import xml.etree.ElementTree
 
-# Soompi uses the same subtitle encryption as crunchyroll
 from .crunchyroll import CrunchyrollIE
 
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    remove_start,
+    xpath_text,
+)
 
-class SoompiIE(CrunchyrollIE):
+
+class SoompiBaseIE(InfoExtractor):
+    def _get_episodes(self, webpage, episode_filter=None):
+        episodes = self._parse_json(
+            self._search_regex(
+                r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'),
+            None)
+        return list(filter(episode_filter, episodes))
+
+
+class SoompiIE(SoompiBaseIE, CrunchyrollIE):
     IE_NAME = 'soompi'
-    _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://tv.soompi.com/en/watch/29235',
         'info_dict': {
@@ -26,84 +40,86 @@ class SoompiIE(CrunchyrollIE):
         },
     }]
 
-    def _get_episodes(self, webpage, episode_filter=None):
-        episodes = json.loads(
-            self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
-        return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
-
-    def _get_subtitles(self, video_id, show_format_xml):
-        subtitles = {}
-        subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
-        subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
+    def _get_episode(self, webpage, video_id):
+        return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0]
 
+    def _get_subtitles(self, config, video_id):
         sub_langs = {}
-        for i in subtitle_info_nodes:
-            sub_langs[i.attrib["id"]] = i.attrib["title"]
+        for subtitle in config.findall('./{default}preload/subtitles/subtitle'):
+            sub_langs[subtitle.attrib['id']] = subtitle.attrib['title']
 
-        for s in subtitle_nodes:
-            lang_code = sub_langs.get(s.attrib["id"], None)
-            if lang_code is None:
+        subtitles = {}
+        for s in config.findall('./{default}preload/subtitle'):
+            lang_code = sub_langs.get(s.attrib['id'])
+            if not lang_code:
+                continue
+            sub_id = s.get('id')
+            data = xpath_text(s, './data', 'data')
+            iv = xpath_text(s, './iv', 'iv')
+            if not id or not iv or not data:
                 continue
-
-            sub_id = int(s.attrib["id"])
-            iv = base64.b64decode(s.find("iv").text)
-            data = base64.b64decode(s.find("data").text)
             subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-
-            subtitles[lang_code] = [{
-                'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
-            }, {
-                'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
-            }]
+            subtitles[lang_code] = self._extract_subtitles(subtitle)
         return subtitles
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            url, video_id, note="Downloading episode page",
-            errnote="Video may not be available for your location")
-        vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage)
-
-        show_meta = json.loads(
-            self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta"))
-        episodes = self._get_episodes(
-            webpage, episode_filter=lambda x: x['id'] == video_id)
-
-        title = episodes[0]["name"]
-        description = episodes[0]["description"]
-        duration = int(episodes[0]["duration"])
-        slug = show_meta["slug"]
+        try:
+            webpage = self._download_webpage(
+                url, video_id, 'Downloading episode page')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                webpage = ee.cause.read()
+                block_message = self._html_search_regex(
+                    r'(?s)<div class="block-message">(.+?)</div>', webpage,
+                    'block message', default=None)
+                if block_message:
+                    raise ExtractorError(block_message, expected=True)
+            raise
 
         formats = []
-        show_format_xml = None
-        for vf in vid_formats:
-            show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \
-                              % (slug, video_id, vf)
-            show_format_xml = self._download_xml(
-                show_format_url, video_id, note="Downloading q%s show xml" % vf)
-            avail_formats = self._extract_m3u8_formats(
-                show_format_xml.find('./{default}preload/stream_info/file').text,
-                video_id, ext="mp4", m3u8_id=vf, preference=int(vf))
-            formats.extend(avail_formats)
+        config = None
+        for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage):
+            config = self._download_xml(
+                'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id),
+                video_id, 'Downloading %s XML' % format_id)
+            m3u8_url = xpath_text(
+                config, './{default}preload/stream_info/file',
+                '%s m3u8 URL' % format_id)
+            if not m3u8_url:
+                continue
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', m3u8_id=format_id))
         self._sort_formats(formats)
 
-        subtitles = self.extract_subtitles(video_id, show_format_xml)
+        episode = self._get_episode(webpage, video_id)
+
+        title = episode['name']
+        description = episode.get('description')
+        duration = int_or_none(episode.get('duration'))
+
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()]
+
+        subtitles = self.extract_subtitles(config, video_id)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnails': thumbnails,
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles
         }
 
 
-class SoompiShowIE(SoompiIE):
+class SoompiShowIE(SoompiBaseIE):
     IE_NAME = 'soompi:show'
-    _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
+    _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
     _TESTS = [{
         'url': 'http://tv.soompi.com/en/shows/liar-game',
         'info_dict': {
@@ -117,14 +133,14 @@ class SoompiShowIE(SoompiIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, show_id, note="Downloading show page")
-        title = self._og_search_title(webpage).replace("SoompiTV | ", "")
+        webpage = self._download_webpage(
+            url, show_id, 'Downloading show page')
+
+        title = remove_start(self._og_search_title(webpage), 'SoompiTV | ')
         description = self._og_search_description(webpage)
 
-        episodes = self._get_episodes(webpage)
-        entries = []
-        for ep in episodes:
-            entries.append(self.url_result(
-                'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
+        entries = [
+            self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi')
+            for episode in self._get_episodes(webpage)]
 
         return self.playlist_result(entries, show_id, title, description)

From 1a5b77dc21384c462e0be86a1638cafd15a6e236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:36:45 +0600
Subject: [PATCH 0743/1515] [crunchyroll] Fix python 3.2

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4ac537a6d..41f0c736d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -76,8 +76,8 @@ class CrunchyrollIE(InfoExtractor):
         self._login()
 
     def _decrypt_subtitles(self, data, iv, id):
-        data = bytes_to_intlist(base64.b64decode(data))
-        iv = bytes_to_intlist(base64.b64decode(iv))
+        data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
+        iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
         id = int(id)
 
         def obfuscate_key_aux(count, modulo, start):

From 5c2191a6053cb5b1210cef68406e8a52e86fd9fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 15:14:10 +0600
Subject: [PATCH 0744/1515] [vgtv] Skip wasLive hds (Closes #5835)

---
 youtube_dl/extractor/vgtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e6ee1e471..654298431 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -107,7 +107,8 @@ class VGTVIE(InfoExtractor):
                 hls_url, video_id, 'mp4', m3u8_id='hls'))
 
         hds_url = streams.get('hds')
-        if hds_url:
+        # wasLive hds are always 404
+        if hds_url and data.get('streamType') != 'wasLive':
             formats.extend(self._extract_f4m_formats(
                 hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 video_id, f4m_id='hds'))

From 4d454c5e4b7ecfae97ff109e05453f43d7cea0a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 15:15:42 +0600
Subject: [PATCH 0745/1515] [vgtv] Check for inactive videos

---
 youtube_dl/extractor/vgtv.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 654298431..472feb7f0 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
 
 
 class VGTVIE(InfoExtractor):
@@ -97,6 +100,10 @@ class VGTVIE(InfoExtractor):
             % (host, video_id, HOST_WEBSITES[host]),
             video_id, 'Downloading media JSON')
 
+        if data.get('status') == 'inactive':
+            raise ExtractorError(
+                'Video %s is no longer available' % video_id, expected=True)
+
         streams = data['streamUrls']
 
         formats = []

From 181c7053e377700c1615bdff2b0fb19235762c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:04:44 +0600
Subject: [PATCH 0746/1515] [YoutubeDL] Make sure all formats have unique
 format_id

---
 youtube_dl/YoutubeDL.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d1953c18f..21d247f23 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1047,6 +1047,8 @@ class YoutubeDL(object):
         if not formats:
             raise ExtractorError('No video formats found!')
 
+        formats_dict = {}
+
         # We check that all the formats have the format and format_id fields
         for i, format in enumerate(formats):
             if 'url' not in format:
@@ -1054,6 +1056,18 @@ class YoutubeDL(object):
 
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
+            format_id = format['format_id']
+            if format_id not in formats_dict:
+                formats_dict[format_id] = []
+            formats_dict[format_id].append(format)
+
+        # Make sure all formats have unique format_id
+        for format_id, ambiguous_formats in formats_dict.items():
+            if len(ambiguous_formats) > 1:
+                for i, format in enumerate(ambiguous_formats):
+                    format['format_id'] = '%s-%d' % (format_id, i)
+
+        for i, format in enumerate(formats):
             if format.get('format') is None:
                 format['format'] = '{id} - {res}{note}'.format(
                     id=format['format_id'],

From b4dd98358f0a68650f6154e8de4e12b8881248aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:12:07 +0600
Subject: [PATCH 0747/1515] [vgtv] Properly handle lives

---
 youtube_dl/extractor/vgtv.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 472feb7f0..f38a72fde 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -62,16 +62,16 @@ class VGTVIE(InfoExtractor):
         },
         {
             # streamType: live
-            'url': 'http://www.vgtv.no/#!/live/100015/direkte-her-kan-du-se-laksen-live-fra-suldalslaagen',
+            'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
             'info_dict': {
-                'id': '100015',
+                'id': '113063',
                 'ext': 'flv',
-                'title': 'DIREKTE: Her kan du se laksen live fra Suldalslågen!',
-                'description': 'md5:9a60cc23fa349f761628924e56eeec2d',
+                'title': 're:^DIREKTE: V75 fra Solvalla [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 0,
-                'timestamp': 1407423348,
-                'upload_date': '20140807',
+                'timestamp': 1432975582,
+                'upload_date': '20150530',
                 'view_count': int,
             },
             'params': {
@@ -105,6 +105,7 @@ class VGTVIE(InfoExtractor):
                 'Video %s is no longer available' % video_id, expected=True)
 
         streams = data['streamUrls']
+        stream_type = data.get('streamType')
 
         formats = []
 
@@ -115,7 +116,7 @@ class VGTVIE(InfoExtractor):
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
-        if hds_url and data.get('streamType') != 'wasLive':
+        if hds_url and stream_type != 'wasLive':
             formats.extend(self._extract_f4m_formats(
                 hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 video_id, f4m_id='hds'))
@@ -143,13 +144,14 @@ class VGTVIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': data['title'],
+            'title': self._live_title(data['title']),
             'description': data['description'],
             'thumbnail': data['images']['main'] + '?t[]=900x506q80',
             'timestamp': data['published'],
             'duration': float_or_none(data['duration'], 1000),
             'view_count': data['displays'],
             'formats': formats,
+            'is_live': True if stream_type == 'live' else False,
         }
 
 

From e6e63e91a70a9c2e4ab92b8afad6fac3b8bede18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:18:11 +0600
Subject: [PATCH 0748/1515] [tf1] Extend _VALID_URL (Closes #5848)

---
 youtube_dl/extractor/tf1.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 656410528..3a68eaa80 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou|www\.tf1)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
@@ -35,6 +35,9 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
         'only_matching': True,
+    }, {
+        'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5196b988971716b9e9c5884d33c757a41aa4548a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 30 May 2015 14:16:18 +0300
Subject: [PATCH 0749/1515] [tubitv] Add new extractor (Closes #5524)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tubitv.py   | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tubitv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f73bf646b..e7e0a55f2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -569,6 +569,7 @@ from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
+from .tubitv import TubiTvIE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
 from .tunein import TuneInIE
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
new file mode 100644
index 000000000..03e971e5e
--- /dev/null
+++ b/youtube_dl/extractor/tubitv.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import codecs
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class TubiTvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video\?id=(?P<id>[0-9]+)'
+    _LOGIN_URL = 'http://tubitv.com/login'
+    _NETRC_MACHINE = 'tubitv'
+    _TEST = {
+        'url': 'http://tubitv.com/video?id=54411&title=The_Kitchen_Musical_-_EP01',
+        'info_dict': {
+            'id': '54411',
+            'ext': 'mp4',
+            'title': 'The Kitchen Musical - EP01',
+            'thumbnail': 're:^https?://.*\.png$',
+            'description': 'md5:37532716166069b353e8866e71fefae7',
+            'duration': 2407,
+        },
+        'params': {
+            'skip_download': 'HLS download',
+        },
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        form_data = {
+            'username': username,
+            'password': password,
+        }
+        payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
+        request = compat_urllib_request.Request(self._LOGIN_URL, payload)
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_page = self._download_webpage(
+            request, None, False, 'Wrong login info')
+        if not re.search(r'id="tubi-logout"', login_page):
+            raise ExtractorError(
+                'Login failed (invalid username/password)', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration'))
+
+        apu = self._search_regex(r"apu='([^']+)'", webpage, 'apu')
+        m3u8_url = codecs.decode(apu, 'rot_13')[::-1]
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+        }

From 1ae7ff771b17d16540aa446aef4f10971465a249 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 30 May 2015 14:33:27 +0300
Subject: [PATCH 0750/1515] [tubitv] Add error message for videos that require
 login (#5524)

---
 youtube_dl/extractor/tubitv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 03e971e5e..2c4b21807 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -59,6 +59,11 @@ class TubiTvIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        if re.search(r"<(?:DIV|div) class='login-required-screen'>", webpage):
+            raise ExtractorError(
+                'This video requires login, use --username and --password '
+                'options to provide account credentials.', expected=True)
+
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)

From 386bdfa698a7f06c43df91913677db3732e29900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 18:29:16 +0600
Subject: [PATCH 0751/1515] [youtube:user] Workaround 35 pages limitation
 (Closes #5778)

---
 youtube_dl/extractor/youtube.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0301682b8..fcdbfe0bc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,6 +1399,26 @@ class YoutubeChannelIE(InfoExtractor):
         channel_id = self._match_id(url)
 
         url = self._TEMPLATE_URL % channel_id
+
+        # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
+        # Workaround by extracting as a playlist if managed to obtain channel playlist URL
+        # otherwise fallback on channel by page extraction
+        channel_page = self._download_webpage(
+            url + '?view=57', channel_id,
+            'Downloading channel page', fatal=False)
+        channel_playlist_id = self._search_regex(
+            [r'<meta itemprop="channelId" content="([^"]+)">',
+             r'data-channel-external-id="([^"]+)"'],
+            channel_page, 'channel id', default=None)
+        if channel_playlist_id and channel_playlist_id.startswith('UC'):
+            playlist_id = 'UU' + channel_playlist_id[2:]
+            channel_playlist = unescapeHTML(self._search_regex(
+                r'href="/?(watch\?v=[0-9A-Za-z_-]{11}&amp;list=%s)"' % playlist_id,
+                channel_page, 'channel playlist URL', default=None))
+            if channel_playlist:
+                return self.url_result(
+                    compat_urlparse.urljoin(url, '/%s' % channel_playlist), 'YoutubePlaylist')
+
         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:

From 9ff811c5cddbf3481fdcd44e97cf3683a925b33f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 May 2015 23:35:55 +0800
Subject: [PATCH 0752/1515] [porn91] PEP8

---
 youtube_dl/extractor/porn91.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index b62eec92d..cdf308f3d 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -5,7 +5,6 @@ import re
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class Porn91IE(InfoExtractor):
@@ -13,13 +12,13 @@ class Porn91IE(InfoExtractor):
     _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
 
     _TEST = {
-            'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-            'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
-            'info_dict': {
-                'id': '7e42283b4f5ab36da134',
-                'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
-                'ext': 'mp4'
-            }
+        'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
+        'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+        'info_dict': {
+            'id': '7e42283b4f5ab36da134',
+            'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+            'ext': 'mp4'
+        }
     }
 
     def _real_extract(self, url):

From 1c2223875664f99325b73fe7765677db9b87e105 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:03:19 +0800
Subject: [PATCH 0753/1515] [porn91] Simplify

---
 youtube_dl/extractor/porn91.py | 38 ++++++++++++++--------------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index cdf308f3d..377ca2c77 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
 
@@ -22,38 +20,34 @@ class Porn91IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
         title = self._search_regex(
-            r'<div id="viewvideo-title">(?P<title>.+?)</div>',
-            webpage, 'title', flags=re.DOTALL)
+            r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')
 
         # get real url
-        n1 = self._search_regex(
-            r'so.addVariable\(\'file\',\'(?P<n1>\d+)\'', webpage, 'n1')
-        n2 = self._search_regex(
-            r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
-        n3 = self._search_regex(
-            r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
+        file_id = self._search_regex(
+            r'so.addVariable\(\'file\',\'(\d+)\'', webpage, 'file id')
+        sec_code = self._search_regex(
+            r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
+        max_vid = self._search_regex(
+            r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
         url_params = compat_urllib_parse.urlencode({
-            'VID': n1,
+            'VID': file_id,
             'mp4': '1',
-            'seccode': n2,
-            'max_vid': n3,
+            'seccode': sec_code,
+            'max_vid': max_vid,
         })
-        t_url = 'http://91porn.com/getfile.php?' + url_params
-        info_cn = self._download_webpage(t_url, video_id, "get real video_url")
-        video_url = self._search_regex(r'file=(?P<url>http.+?)&', info_cn, 'url')
+        info_cn = self._download_webpage(
+            'http://91porn.com/getfile.php?' + url_params, video_id,
+            "get real video url")
+        video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
-        # construct info
-        info = {
+        return {
             'id': video_id,
             'title': title,
             'url': video_url,
         }
-
-        return info

From a80601f8d9789e27c0a916e63d7192c3f398d5d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:20:37 +0800
Subject: [PATCH 0754/1515] [porn91] Extract more info

---
 youtube_dl/extractor/porn91.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 377ca2c77..ea1efc71b 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
 
 
 class Porn91IE(InfoExtractor):
@@ -15,7 +19,8 @@ class Porn91IE(InfoExtractor):
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
-            'ext': 'mp4'
+            'ext': 'mp4',
+            'duration': 431,
         }
     }
 
@@ -46,8 +51,16 @@ class Porn91IE(InfoExtractor):
             "get real video url")
         video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
+        duration = parse_duration(self._search_regex(
+            r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+
+        comment_count = int_or_none(self._search_regex(
+            r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
+
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
+            'duration': duration,
+            'comment_count': comment_count,
         }

From d05a1dbe7013d6314ec477b50d864726e509a872 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:26:12 +0800
Subject: [PATCH 0755/1515] [porn91] Catch daily limit error

---
 youtube_dl/extractor/porn91.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index ea1efc71b..c119c7e94 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -29,6 +30,10 @@ class Porn91IE(InfoExtractor):
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
+
+        if '作为游客，你每天只可观看10个视频' in webpage:
+            raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
+
         title = self._search_regex(
             r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')

From a2d971309b75c79f3f688a0c381707d828cb1026 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:31:18 +0800
Subject: [PATCH 0756/1515] [porn91] Use single quotes

---
 youtube_dl/extractor/porn91.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index c119c7e94..72d1b2718 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -29,7 +29,7 @@ class Porn91IE(InfoExtractor):
         video_id = self._match_id(url)
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
-        webpage = self._download_webpage(url, video_id, "get HTML content")
+        webpage = self._download_webpage(url, video_id, 'get HTML content')
 
         if '作为游客，你每天只可观看10个视频' in webpage:
             raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
@@ -53,7 +53,7 @@ class Porn91IE(InfoExtractor):
         })
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
-            "get real video url")
+            'get real video url')
         video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
         duration = parse_duration(self._search_regex(

From 931bc3c3a719fe33101c05b9fdc4e6ad8eb08bdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 22:52:02 +0600
Subject: [PATCH 0757/1515] [YoutubeDL] Do not loose request method information

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21d247f23..5fc8754c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -49,6 +49,7 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
+    HEADRequest,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
@@ -1720,7 +1721,8 @@ class YoutubeDL(object):
             if req_is_string:
                 req = url_escaped
             else:
-                req = compat_urllib_request.Request(
+                req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+                req = req_type(
                     url_escaped, data=req.data, headers=req.headers,
                     origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
 

From 339516072be6865bf7e9316be81704ae69296c6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 23:16:14 +0600
Subject: [PATCH 0758/1515] [extractor/generic] Unescape video_id and title
 extracted from URL

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9a7b0d25d..c9c92d686 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -894,7 +894,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
+            video_id = os.path.splitext(compat_urllib_parse.unquote(url.rstrip('/').split('/')[-1]))[0]
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -927,7 +927,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(url_basename(url))[0],
+                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
                 'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
@@ -953,7 +953,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(url_basename(url))[0],
+                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,

From 58bde34a236ff98f25fc109a94b3d393f0bbc9ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 00:44:54 +0600
Subject: [PATCH 0759/1515] [extractor/generic] Force Accept-Encoding to any
 for extraction pass

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c9c92d686..ec1d9abbe 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
     compat_xml_parse_error,
+    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
@@ -916,7 +917,9 @@ class GenericIE(InfoExtractor):
 
         full_response = None
         if head_response is False:
-            full_response = self._request_webpage(url, video_id)
+            request = compat_urllib_request.Request(url)
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
             head_response = full_response
 
         # Check for direct link to a video
@@ -941,7 +944,17 @@ class GenericIE(InfoExtractor):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if not full_response:
-            full_response = self._request_webpage(url, video_id)
+            request = compat_urllib_request.Request(url)
+            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
+            # making it impossible to download only chunk of the file (yet we need only 512kB to
+            # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
+            # that will always result in downloading the whole file that is not desirable.
+            # Therefore for extraction pass we have to override Accept-Encoding to any in order
+            # to accept raw bytes and being able to download only a chunk.
+            # It may probably better to solve this by checking Content-Type for application/octet-stream
+            # after HEAD request finishes, but not sure if we can rely on this.
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
 
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!

From 1ddb9456c4a63a207ec40bd74cdf0b36d8c68409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 01:23:58 +0600
Subject: [PATCH 0760/1515] [extractor/generic] Use compat_urllib_parse_unquote
 for unquoting video_id and title from URL

---
 youtube_dl/extractor/generic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec1d9abbe..d9116ce10 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -9,9 +9,10 @@ from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
-    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
@@ -895,7 +896,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = os.path.splitext(compat_urllib_parse.unquote(url.rstrip('/').split('/')[-1]))[0]
+            video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -930,7 +931,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
+                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
@@ -966,7 +967,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
+                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,

From a074e922967fa571d4f1abb1773c711747060f00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:13:24 +0600
Subject: [PATCH 0761/1515] [extractor/generic] Add test for large compressed
 media

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d9116ce10..737141f95 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -138,6 +138,20 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20100513',
             }
         },
+        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        {
+            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
+            'md5': '128c42e68b13950268b648275386fc74',
+            'info_dict': {
+                'id': 'FictionJunction-Parallel_Hearts',
+                'ext': 'flac',
+                'title': 'FictionJunction-Parallel_Hearts',
+                'upload_date': '20140522',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',

From c5fa81fe81ce05cd81c20ff4ea6dac3dccdcbf9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:22:29 +0600
Subject: [PATCH 0762/1515] [extractor/generic] Put all direct link tests near
 to each other for better navigation

---
 youtube_dl/extractor/generic.py | 182 ++++++++++++++++----------------
 1 file changed, 91 insertions(+), 91 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 737141f95..8f2e53063 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,97 @@ class GenericIE(InfoExtractor):
     _VALID_URL = r'.*'
     IE_NAME = 'generic'
     _TESTS = [
+        # Direct link to a video
+        {
+            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
+            'info_dict': {
+                'id': 'trailer',
+                'ext': 'mp4',
+                'title': 'trailer',
+                'upload_date': '20100513',
+            }
+        },
+        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        {
+            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
+            'md5': '128c42e68b13950268b648275386fc74',
+            'info_dict': {
+                'id': 'FictionJunction-Parallel_Hearts',
+                'ext': 'flac',
+                'title': 'FictionJunction-Parallel_Hearts',
+                'upload_date': '20140522',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
+        # Direct download with broken HEAD
+        {
+            'url': 'http://ai-radio.org:8000/radio.opus',
+            'info_dict': {
+                'id': 'radio',
+                'ext': 'opus',
+                'title': 'radio',
+            },
+            'params': {
+                'skip_download': True,  # infinite live stream
+            },
+            'expected_warnings': [
+                r'501.*Not Implemented'
+            ],
+        },
+        # Direct link with incorrect MIME type
+        {
+            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+            'md5': '4ccbebe5f36706d85221f204d7eb5913',
+            'info_dict': {
+                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+                'id': '5_Lennart_Poettering_-_Systemd',
+                'ext': 'webm',
+                'title': '5_Lennart_Poettering_-_Systemd',
+                'upload_date': '20141120',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
+        # RSS feed
+        {
+            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+            'info_dict': {
+                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+                'title': 'Zero Punctuation',
+                'description': 're:.*groundbreaking video review series.*'
+            },
+            'playlist_mincount': 11,
+        },
+        # RSS feed with enclosure
+        {
+            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
+            'info_dict': {
+                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+                'ext': 'm4v',
+                'upload_date': '20150228',
+                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+            }
+        },
+        # google redirect
+        {
+            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
+            'info_dict': {
+                'id': 'cmQHVoWB5FY',
+                'ext': 'mp4',
+                'upload_date': '20130224',
+                'uploader_id': 'TheVerge',
+                'description': 're:^Chris Ziegler takes a look at the\.*',
+                'uploader': 'The Verge',
+                'title': 'First Firefox OS phones side-by-side',
+            },
+            'params': {
+                'skip_download': False,
+            }
+        },
         {
             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
@@ -127,31 +218,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
-        # Direct link to a video
-        {
-            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
-            'info_dict': {
-                'id': 'trailer',
-                'ext': 'mp4',
-                'title': 'trailer',
-                'upload_date': '20100513',
-            }
-        },
-        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
-        {
-            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
-            'md5': '128c42e68b13950268b648275386fc74',
-            'info_dict': {
-                'id': 'FictionJunction-Parallel_Hearts',
-                'ext': 'flac',
-                'title': 'FictionJunction-Parallel_Hearts',
-                'upload_date': '20140522',
-            },
-            'expected_warnings': [
-                'URL could be a direct video link, returning it as such.'
-            ]
-        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
@@ -176,22 +242,6 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
-        # google redirect
-        {
-            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
-            'info_dict': {
-                'id': 'cmQHVoWB5FY',
-                'ext': 'mp4',
-                'upload_date': '20130224',
-                'uploader_id': 'TheVerge',
-                'description': 're:^Chris Ziegler takes a look at the\.*',
-                'uploader': 'The Verge',
-                'title': 'First Firefox OS phones side-by-side',
-            },
-            'params': {
-                'skip_download': False,
-            }
-        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
@@ -423,16 +473,6 @@ class GenericIE(InfoExtractor):
                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
             }
         },
-        # RSS feed
-        {
-            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
-            'info_dict': {
-                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
-                'title': 'Zero Punctuation',
-                'description': 're:.*groundbreaking video review series.*'
-            },
-            'playlist_mincount': 11,
-        },
         # Multiple brightcove videos
         # https://github.com/rg3/youtube-dl/issues/2283
         {
@@ -486,21 +526,6 @@ class GenericIE(InfoExtractor):
                 'uploader': 'thoughtworks.wistia.com',
             },
         },
-        # Direct download with broken HEAD
-        {
-            'url': 'http://ai-radio.org:8000/radio.opus',
-            'info_dict': {
-                'id': 'radio',
-                'ext': 'opus',
-                'title': 'radio',
-            },
-            'params': {
-                'skip_download': True,  # infinite live stream
-            },
-            'expected_warnings': [
-                r'501.*Not Implemented'
-            ],
-        },
         # Soundcloud embed
         {
             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
@@ -532,21 +557,6 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 2,
         },
-        # Direct link with incorrect MIME type
-        {
-            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
-            'md5': '4ccbebe5f36706d85221f204d7eb5913',
-            'info_dict': {
-                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
-                'id': '5_Lennart_Poettering_-_Systemd',
-                'ext': 'webm',
-                'title': '5_Lennart_Poettering_-_Systemd',
-                'upload_date': '20141120',
-            },
-            'expected_warnings': [
-                'URL could be a direct video link, returning it as such.'
-            ]
-        },
         # Cinchcast embed
         {
             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
@@ -705,16 +715,6 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
-        # RSS feed with enclosure
-        {
-            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
-            'info_dict': {
-                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
-                'ext': 'm4v',
-                'upload_date': '20150228',
-                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
-            }
-        },
         # Crooks and Liars embed
         {
             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',

From c5138a7ce49db19b64adc11d81384595b966a7a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:36:20 +0600
Subject: [PATCH 0763/1515] [extractor/generic] Clarify test comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f2e53063..96ca398de 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,7 +59,7 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20100513',
             }
         },
-        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        # Direct link to media delivered compressed (until Accept-Encoding is *)
         {
             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
             'md5': '128c42e68b13950268b648275386fc74',

From d2a9de78dfc629aaaaf8a2a30432d5f02c949e9a Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Sat, 30 May 2015 20:50:22 +0000
Subject: [PATCH 0764/1515] [youtube] Construct a playlist URL in case the page
 is missing one

This fixes jumping from user/channel -> playlist for some users like
https://www.youtube.com/user/BitcoinFoundation

This also removes the superfluous log message
"add --no-playlist to just download video VIDEOID"
when downloading a user/channel.
---
 youtube_dl/extractor/youtube.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fcdbfe0bc..aacb999ce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1412,12 +1412,8 @@ class YoutubeChannelIE(InfoExtractor):
             channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
-            channel_playlist = unescapeHTML(self._search_regex(
-                r'href="/?(watch\?v=[0-9A-Za-z_-]{11}&amp;list=%s)"' % playlist_id,
-                channel_page, 'channel playlist URL', default=None))
-            if channel_playlist:
-                return self.url_result(
-                    compat_urlparse.urljoin(url, '/%s' % channel_playlist), 'YoutubePlaylist')
+            return self.url_result(
+                compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
 
         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)

From eb47569f8a6017190d73429b3ef54c1ffaf201dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 03:00:13 +0600
Subject: [PATCH 0765/1515] [tvigle] Add support for m3u8

---
 youtube_dl/extractor/tvigle.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 102362b29..4e95bd30f 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
+    int_or_none,
     parse_age_limit,
 )
 
@@ -24,17 +25,17 @@ class TvigleIE(InfoExtractor):
                 'display_id': 'sokrat',
                 'ext': 'flv',
                 'title': 'Сократ',
-                'description': 'md5:a05bd01be310074d5833efc6743be95e',
+                'description': 'md5:d6b92ffb7217b4b8ebad2e7665253c17',
                 'duration': 6586,
-                'age_limit': 0,
+                'age_limit': 12,
             },
         },
         {
             'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
-            'md5': 'd9012d7c7c598fe7a11d7fb46dc1f574',
+            'md5': 'e7efe5350dd5011d0de6550b53c3ba7b',
             'info_dict': {
                 'id': '5142516',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Ведущий телепрограммы «60 минут» (США) о Владимире Высоцком',
                 'description': 'md5:027f7dc872948f14c96d19b4178428a4',
                 'duration': 186.080,
@@ -54,7 +55,7 @@ class TvigleIE(InfoExtractor):
         if not video_id:
             webpage = self._download_webpage(url, display_id)
             video_id = self._html_search_regex(
-                r'<li class="video-preview current_playing" id="(\d+)">',
+                r'class="video-preview current_playing" id="(\d+)">',
                 webpage, 'video id')
 
         video_data = self._download_json(
@@ -70,13 +71,19 @@ class TvigleIE(InfoExtractor):
 
         formats = []
         for vcodec, fmts in item['videos'].items():
-            for quality, video_url in fmts.items():
+            for format_id, video_url in fmts.items():
+                if format_id == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=vcodec))
+                    continue
+                height = self._search_regex(
+                    r'^(\d+)[pP]$', format_id, 'height', default=None)
                 formats.append({
                     'url': video_url,
-                    'format_id': '%s-%s' % (vcodec, quality),
+                    'format_id': '%s-%s' % (vcodec, format_id),
                     'vcodec': vcodec,
-                    'height': int(quality[:-1]),
-                    'filesize': item['video_files_size'][vcodec][quality],
+                    'height': int_or_none(height),
+                    'filesize': item['video_files_size'][vcodec][format_id],
                 })
         self._sort_formats(formats)
 

From 7584e38ce4e98e0e9abca146a513d215701308e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 03:01:41 +0600
Subject: [PATCH 0766/1515] [tvigle] Modernize

---
 youtube_dl/extractor/tvigle.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 4e95bd30f..a85693888 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -64,8 +64,8 @@ class TvigleIE(InfoExtractor):
         item = video_data['playlist']['items'][0]
 
         title = item['title']
-        description = item['description']
-        thumbnail = item['thumbnail']
+        description = item.get('description')
+        thumbnail = item.get('thumbnail')
         duration = float_or_none(item.get('durationMilliseconds'), 1000)
         age_limit = parse_age_limit(item.get('ageRestrictions'))
 
@@ -83,7 +83,7 @@ class TvigleIE(InfoExtractor):
                     'format_id': '%s-%s' % (vcodec, format_id),
                     'vcodec': vcodec,
                     'height': int_or_none(height),
-                    'filesize': item['video_files_size'][vcodec][format_id],
+                    'filesize': int_or_none(item.get('video_files_size', {}).get(vcodec, {}).get(format_id)),
                 })
         self._sort_formats(formats)
 

From df15ef8dab6df79eb076b3d06b3948917763ac3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 04:05:09 +0600
Subject: [PATCH 0767/1515] [YoutubeDL] Tweak select_format for video only
 media

---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5fc8754c6..aa6ec9d9a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -924,8 +924,9 @@ class YoutubeDL(object):
                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
             if audiovideo_formats:
                 return audiovideo_formats[format_idx]
-            # for audio only urls, select the best/worst audio format
-            elif all(f.get('acodec') != 'none' for f in available_formats):
+            # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
+            elif (all(f.get('acodec') != 'none' for f in available_formats) or
+                  all(f.get('vcodec') != 'none' for f in available_formats)):
                 return available_formats[format_idx]
         elif format_spec == 'bestaudio':
             audio_formats = [

From 96b9690985e9b9f4e50fde10bbc92e1a72df64e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 04:05:26 +0600
Subject: [PATCH 0768/1515] [imgur] Improve extraction

---
 youtube_dl/extractor/imgur.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index fe5d95e2c..d692ea79a 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
@@ -12,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -34,7 +35,8 @@ class ImgurIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            compat_urlparse.urljoin(url, video_id), video_id)
 
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',

From 47fd8c2f761c2073744cb041f9eccb7ed10f2470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jun 2015 00:04:36 +0600
Subject: [PATCH 0769/1515] [patreon] Fix embeds extraction (Closes #5862)

---
 youtube_dl/extractor/patreon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index f179ea200..6cdc2638b 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -87,7 +87,7 @@ class PatreonIE(InfoExtractor):
             r'<div class="attach"><a target="_blank" href="([^"]+)">',
             webpage, 'attachment URL', default=None)
         embed = self._html_search_regex(
-            r'<div id="watchCreation">\s*<iframe class="embedly-embed" src="([^"]+)"',
+            r'<div[^>]+id="watchCreation"[^>]*>\s*<iframe[^>]+src="([^"]+)"',
             webpage, 'embedded URL', default=None)
 
         if attach_fn is not None:

From 4053ee9104cd7669f749a267dc2c2a1725ca188b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Jun 2015 14:43:20 +0800
Subject: [PATCH 0770/1515] Credit @PeterDing for 91porn extractor (#5830)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3410e1fb9..bf2a25cb8 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -126,3 +126,4 @@ Matthias Küch
 Julian Richen
 Ping O.
 Mister Hat
+Peter Ding

From 866b296d0f156831cceccc967c34382a90b77422 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 1 Jun 2015 16:11:19 +0300
Subject: [PATCH 0771/1515] [aftonbladet] Fix extraction and update _VALID_URL
 (Fixes #5863)

---
 youtube_dl/extractor/aftonbladet.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index a117502bc..e0518cf26 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -6,11 +6,11 @@ from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.aftonbladet\.se/webbtv.+?(?P<id>article[0-9]+)\.ab(?:$|[?#])'
+    _VALID_URL = r'http://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://tv.aftonbladet.se/webbtv/nyheter/vetenskap/rymden/article36015.ab',
+        'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
         'info_dict': {
-            'id': 'article36015',
+            'id': '36015',
             'ext': 'mp4',
             'title': 'Vulkanutbrott i rymden - nu släpper NASA bilderna',
             'description': 'Jupiters måne mest aktiv av alla himlakroppar',
@@ -25,8 +25,9 @@ class AftonbladetIE(InfoExtractor):
 
         # find internal video meta data
         meta_url = 'http://aftonbladet-play.drlib.aptoma.no/video/%s.json'
-        internal_meta_id = self._html_search_regex(
-            r'data-aptomaId="([\w\d]+)"', webpage, 'internal_meta_id')
+        player_config = self._parse_json(self._html_search_regex(
+            r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
+        internal_meta_id = player_config['videoId']
         internal_meta_url = meta_url % internal_meta_id
         internal_meta_json = self._download_json(
             internal_meta_url, video_id, 'Downloading video meta data')

From 923e79e2e4d9cc0c24496614aab520737cdc89ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 00:53:04 +0600
Subject: [PATCH 0772/1515] [nova] Add extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/nova.py     | 135 +++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 youtube_dl/extractor/nova.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4dc07efe0..67eb96057 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -352,6 +352,7 @@ from .ninegag import NineGagIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
+from .nova import NovaIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
 from .nowtv import NowTVIE
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
new file mode 100644
index 000000000..e93a7ffa8
--- /dev/null
+++ b/youtube_dl/extractor/nova.py
@@ -0,0 +1,135 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class NovaIE(InfoExtractor):
+    IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+)(?:\.html|/?)'
+    _TESTS = [{
+        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
+        'info_dict': {
+            'id': '1608920',
+            'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
+            'ext': 'flv',
+            'title': 'Duel: Michal Hrdlička a Petr Suchoň',
+            'description': 'md5:d0cc509858eee1b1374111c588c6f5d5',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html',
+        'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
+        'info_dict': {
+            'id': '1757139',
+            'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci',
+            'ext': 'mp4',
+            'title': 'Podzemní nemocnice v pražské Krči',
+            'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        }
+    }, {
+        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/',
+        'info_dict': {
+            'id': '1756825',
+            'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
+            'ext': 'flv',
+            'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
+            'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://doma.nova.cz/clanek/zdravi/prijdte-se-zapsat-do-registru-kostni-drene-jiz-ve-stredu-3-cervna.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        site = mobj.group('site')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            [r"(?:media|video_id)\s*:\s*'(\d+)'",
+             r'media=(\d+)',
+             r'id="article_video_(\d+)"',
+             r'id="player_(\d+)"'],
+            webpage, 'video id')
+
+        config_url = self._search_regex(
+            r'src="(http://tn\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
+            webpage, 'config url', default=None)
+
+        if not config_url:
+            DEFAULT_SITE_ID = '23000'
+            SITES = {
+                'tvnoviny': DEFAULT_SITE_ID,
+                'novaplus': DEFAULT_SITE_ID,
+                'vymena': DEFAULT_SITE_ID,
+                'krasna': DEFAULT_SITE_ID,
+                'fanda': '30',
+                'tn': '30',
+                'doma': '30',
+            }
+
+            site_id = self._search_regex(
+                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(site, DEFAULT_SITE_ID)
+
+            config_url = ('http://tn.nova.cz/bin/player/videojs/config.php?site=%s&media=%s&jsVar=vjsconfig'
+                          % (site_id, video_id))
+
+        config = self._download_json(
+            config_url, display_id,
+            'Downloading config JSON',
+            transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s))
+
+        mediafile = config['mediafile']
+        video_url = mediafile['src']
+
+        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
+        if m:
+            formats = [{
+                'url': m.group('url'),
+                'app': m.group('app'),
+                'play_path': m.group('playpath'),
+                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
+                'ext': 'flv',
+            }]
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+        self._sort_formats(formats)
+
+        title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = config.get('poster')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 60158217ef8da5f44ef316e50c8a5e2ac2e202c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 00:57:08 +0600
Subject: [PATCH 0773/1515] [nova] Add tv test

---
 youtube_dl/extractor/nova.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index e93a7ffa8..4e999b237 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -60,6 +60,9 @@ class NovaIE(InfoExtractor):
     }, {
         'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
         'only_matching': True,
+    }, {
+        'url': 'http://tv.nova.cz/clanek/novinky/zivot-je-zivot-bondovsky-trailer.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9f4b9118ccaef5cd7c414a78c5622968e8c3343f Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:47:52 +0300
Subject: [PATCH 0774/1515] [nova] Fix display_id extraction bug

Make id group non-greedy so that .html is not included in it.
---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 4e999b237..1dd18511e 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+)(?:\.html|/?)'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/?)$'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
         'info_dict': {

From 9464a194dbf48989c486fa2de9e1aebc59e28ed4 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:52:39 +0300
Subject: [PATCH 0775/1515] [nova] Fix extension extraction bug

Replace the hardcoded flv with determine_ext. Let rtmpdump parse the url.
---
 youtube_dl/extractor/nova.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 1dd18511e..fd5f9cb0e 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class NovaIE(InfoExtractor):
@@ -39,7 +40,7 @@ class NovaIE(InfoExtractor):
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
             'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
@@ -108,21 +109,8 @@ class NovaIE(InfoExtractor):
 
         mediafile = config['mediafile']
         video_url = mediafile['src']
-
-        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
-        if m:
-            formats = [{
-                'url': m.group('url'),
-                'app': m.group('app'),
-                'play_path': m.group('playpath'),
-                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
-                'ext': 'flv',
-            }]
-        else:
-            formats = [{
-                'url': video_url,
-            }]
-        self._sort_formats(formats)
+        ext = determine_ext(video_url)
+        video_url = video_url.replace('&{}:'.format(ext), '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = self._og_search_description(webpage)
@@ -134,5 +122,6 @@ class NovaIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'formats': formats,
+            'url': video_url,
+            'ext': ext,
         }

From fcb04bcaca1b83cd3f13f494d7d775e35e0b6182 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:55:41 +0300
Subject: [PATCH 0776/1515] [nova] Extract upload_date in some cases

---
 youtube_dl/extractor/nova.py | 39 ++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index fd5f9cb0e..30c64aaf8 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -49,6 +49,33 @@ class NovaIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
+        'info_dict': {
+            'id': '1756858',
+            'ext': 'mp4',
+            'title': 'Televizní noviny - 30. 5. 2015',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+            'upload_date': '20150530',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
+        'info_dict': {
+            'id': '1753621',
+            'ext': 'mp4',
+            'title': 'Zaklínač 3: Divoký hon',
+            'description': 're:.*Pokud se stejně jako my nemůžete.*',
+            'thumbnail': 're:https?://.*\.jpg(\?.*)?',
+            'upload_date': '20150521',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
     }, {
         'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
         'only_matching': True,
@@ -116,11 +143,23 @@ class NovaIE(InfoExtractor):
         description = self._og_search_description(webpage)
         thumbnail = config.get('poster')
 
+        mobj = None
+        if site == 'novaplus':
+            mobj = re.search(r'(?P<day>\d{1,2})-(?P<month>\d{1,2})-(?P<year>\d{4})$', display_id)
+        if site == 'fanda':
+            mobj = re.search(
+                r'<span class="date_time">(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4})\b', webpage)
+        if mobj:
+            upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day')))
+        else:
+            upload_date = None
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
+            'upload_date': upload_date,
             'thumbnail': thumbnail,
             'url': video_url,
             'ext': ext,

From 34c0f95db273ac5e7a7f8a6d23a3f90ceadf4695 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:56:36 +0300
Subject: [PATCH 0777/1515] [nova] Remove html tags from description

---
 youtube_dl/extractor/nova.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 30c64aaf8..140312f9c 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import clean_html, determine_ext
 
 
 class NovaIE(InfoExtractor):
@@ -42,7 +42,7 @@ class NovaIE(InfoExtractor):
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
             'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
+            'description': 'md5:dc24e50be5908df83348e50d1431295e',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         },
         'params': {
@@ -140,7 +140,7 @@ class NovaIE(InfoExtractor):
         video_url = video_url.replace('&{}:'.format(ext), '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
+        description = clean_html(self._og_search_description(webpage, default=None))
         thumbnail = config.get('poster')
 
         mobj = None

From a00234f1c517d077a237da576be638fef980d79e Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:57:03 +0300
Subject: [PATCH 0778/1515] [nova] Minor style improvement

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 140312f9c..10957e5fa 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -132,7 +132,7 @@ class NovaIE(InfoExtractor):
         config = self._download_json(
             config_url, display_id,
             'Downloading config JSON',
-            transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s))
+            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
 
         mediafile = config['mediafile']
         video_url = mediafile['src']

From bc03e58565d99677a643e0a058d25c7ee9b265d6 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 13:16:58 +0300
Subject: [PATCH 0779/1515] [iprima] Update

---
 youtube_dl/extractor/iprima.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 8529bedfc..f3536893a 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/[^?#]+/(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/([^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/particka/particka-92',
@@ -23,7 +23,7 @@ class IPrimaIE(InfoExtractor):
             'id': '39152',
             'ext': 'flv',
             'title': 'Partička (92)',
-            'description': 'md5:3740fda51464da35a2d4d0670b8e4fd6',
+            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
         },
         'params': {
@@ -35,13 +35,16 @@ class IPrimaIE(InfoExtractor):
             'id': '9718337',
             'ext': 'flv',
             'title': 'Tchibo Partička - Jarní móda',
-            'description': 'md5:589f8f59f414220621ff8882eb3ce7be',
+            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,  # requires rtmpdump
         },
         'skip': 'Do not have permission to access this page',
+    }, {
+        'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -102,7 +105,7 @@ class IPrimaIE(InfoExtractor):
 
         return {
             'id': real_id,
-            'title': self._og_search_title(webpage),
+            'title': self._og_search_title(webpage).replace(' | Prima PLAY', ''),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'description': self._og_search_description(webpage),

From b5597738d4de35fd6f2be7bf1cb6a32c754d873f Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 17:28:14 +0300
Subject: [PATCH 0780/1515] [iprima] Comply with review

---
 youtube_dl/extractor/iprima.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index f3536893a..502507551 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -11,11 +11,12 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    remove_end,
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/([^/]+/)*(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/particka/particka-92',
@@ -105,7 +106,7 @@ class IPrimaIE(InfoExtractor):
 
         return {
             'id': real_id,
-            'title': self._og_search_title(webpage).replace(' | Prima PLAY', ''),
+            'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'description': self._og_search_description(webpage),

From d23da75b32e02963f988bad962b3f5259e4a6d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 21:10:18 +0600
Subject: [PATCH 0781/1515] [iprima] Fix description extraction

`og:description` does not contain actual description anymore.
---
 youtube_dl/extractor/iprima.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 502507551..821c8ec10 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -24,7 +24,7 @@ class IPrimaIE(InfoExtractor):
             'id': '39152',
             'ext': 'flv',
             'title': 'Partička (92)',
-            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
+            'description': 'md5:74e9617e51bca67c3ecfb2c6f9766f45',
             'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
         },
         'params': {
@@ -36,13 +36,11 @@ class IPrimaIE(InfoExtractor):
             'id': '9718337',
             'ext': 'flv',
             'title': 'Tchibo Partička - Jarní móda',
-            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,  # requires rtmpdump
         },
-        'skip': 'Do not have permission to access this page',
     }, {
         'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
         'only_matching': True,
@@ -109,5 +107,7 @@ class IPrimaIE(InfoExtractor):
             'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
-            'description': self._og_search_description(webpage),
+            'description': self._search_regex(
+                r'<p[^>]+itemprop="description"[^>]*>([^<]+)',
+                webpage, 'description', default=None),
         }

From 4b5fe1349f5568f3b9b939520db0a1ddc598b4b3 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 18:15:05 +0300
Subject: [PATCH 0782/1515] [nova] Comply with review

---
 youtube_dl/extractor/nova.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 10957e5fa..4a2d76506 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import clean_html, determine_ext
+from ..utils import (
+    clean_html,
+    determine_ext,
+    unified_strdate,
+)
 
 
 class NovaIE(InfoExtractor):
@@ -143,14 +147,12 @@ class NovaIE(InfoExtractor):
         description = clean_html(self._og_search_description(webpage, default=None))
         thumbnail = config.get('poster')
 
-        mobj = None
         if site == 'novaplus':
-            mobj = re.search(r'(?P<day>\d{1,2})-(?P<month>\d{1,2})-(?P<year>\d{4})$', display_id)
-        if site == 'fanda':
-            mobj = re.search(
-                r'<span class="date_time">(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4})\b', webpage)
-        if mobj:
-            upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day')))
+            upload_date = unified_strdate(self._search_regex(
+                r'(\d{1,2}-\d{1,2}-\d{4})$', display_id, 'upload date', default=None))
+        elif site == 'fanda':
+            upload_date = unified_strdate(self._search_regex(
+                r'<span class="date_time">(\d{1,2}\.\d{1,2}\.\d{4})', webpage, 'upload date', default=None))
         else:
             upload_date = None
 

From 08b7968e2873b45dafe465ec04541db8fcd4967d Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 17:49:15 +0300
Subject: [PATCH 0783/1515] [nova] Fix display_id extraction bug

---
 youtube_dl/extractor/nova.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 4a2d76506..8360a65d9 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -13,9 +13,9 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/?)$'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
-        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
+        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
             'id': '1608920',
             'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
@@ -29,7 +29,7 @@ class NovaIE(InfoExtractor):
             'skip_download': True,
         }
     }, {
-        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html',
+        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
         'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
         'info_dict': {
             'id': '1757139',
@@ -40,7 +40,7 @@ class NovaIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         }
     }, {
-        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/',
+        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove',
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',

From b0cda32f726443d464a68a34b22a2e02ef8b29b0 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 18:17:33 +0300
Subject: [PATCH 0784/1515] [nova] Fix Python 2.6 compatability issue

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 8360a65d9..7e3498eea 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -141,7 +141,7 @@ class NovaIE(InfoExtractor):
         mediafile = config['mediafile']
         video_url = mediafile['src']
         ext = determine_ext(video_url)
-        video_url = video_url.replace('&{}:'.format(ext), '')
+        video_url = video_url.replace('&%s:' % ext, '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = clean_html(self._og_search_description(webpage, default=None))

From fa971259e69a8031c384754b6238cfff71bea773 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 19:09:47 +0300
Subject: [PATCH 0785/1515] [nova] Add a comment about html in description

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 7e3498eea..85253b6ed 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -46,7 +46,7 @@ class NovaIE(InfoExtractor):
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
             'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:dc24e50be5908df83348e50d1431295e',
+            'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         },
         'params': {

From 23dd1fc74c36329ed40855301ac499a0ad2a0009 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 10:21:03 +0800
Subject: [PATCH 0786/1515] [vidme] Always use the non-embedded page

For example, https://vid.me/Wmur contains more information than
https://vid.me/e/Wmur
---
 youtube_dl/extractor/vidme.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index bd953fb4c..e0b55078b 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vid.me/QNB',
         'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
         'info_dict': {
@@ -23,9 +23,14 @@ class VidmeIE(InfoExtractor):
             'upload_date': '20140725',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-    }
+    }, {
+        # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
+        'url': 'https://vid.me/e/Wmur',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
+        url = url.replace('vid.me/e/', 'vid.me/')
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 

From 8f9478412424b87e4fb77be53d239c13932b078a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 10:26:39 +0800
Subject: [PATCH 0787/1515] [tumblr] Detect vid.me embeds (fixes #5883)

---
 youtube_dl/extractor/tumblr.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 828c808a6..e6218808f 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -28,6 +28,17 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
+        'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
+        'info_dict': {
+            'id': 'Wmur',
+            'ext': 'mp4',
+            'title': 'naked smoking & stretching',
+            'upload_date': '20150506',
+            'timestamp': 1430931613,
+        },
+        'add_ie': ['Vidme'],
     }]
 
     def _real_extract(self, url):
@@ -38,6 +49,12 @@ class TumblrIE(InfoExtractor):
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage = self._download_webpage(url, video_id)
 
+        vid_me_embed_url = self._search_regex(
+            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
+            webpage, 'vid.me embed', default=None)
+        if vid_me_embed_url is not None:
+            return self.url_result(vid_me_embed_url, 'Vidme')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From 687cb3ad35ac49f1053c1ea52e3b6db18b3aa1cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:47:11 +0600
Subject: [PATCH 0788/1515] [24video] Fix uploader extraction

---
 youtube_dl/extractor/twentyfourvideo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 67e8bfea0..c1ee1decc 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -15,7 +15,7 @@ class TwentyFourVideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.24video.net/video/view/1044982',
-            'md5': '48dd7646775690a80447a8dca6a2df76',
+            'md5': 'd041af8b5b4246ea466226a0d6693345',
             'info_dict': {
                 'id': '1044982',
                 'ext': 'mp4',
@@ -54,7 +54,7 @@ class TwentyFourVideoIE(InfoExtractor):
             webpage, 'upload date'))
 
         uploader = self._html_search_regex(
-            r'Загрузил\s*<a href="/jsecUser/movies/[^"]+" class="link">([^<]+)</a>',
+            r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
             webpage, 'uploader', fatal=False)
 
         view_count = int_or_none(self._html_search_regex(

From 15b74b94beb8720a4f6d7ee076c123dd8ae05309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:52:47 +0600
Subject: [PATCH 0789/1515] [tvigle] Capture error message

---
 youtube_dl/extractor/tvigle.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index a85693888..aa07c8251 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     float_or_none,
     int_or_none,
     parse_age_limit,
@@ -63,6 +64,13 @@ class TvigleIE(InfoExtractor):
 
         item = video_data['playlist']['items'][0]
 
+        videos = item.get('videos')
+
+        error_message = item.get('errorMessage')
+        if not videos and error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
         title = item['title']
         description = item.get('description')
         thumbnail = item.get('thumbnail')

From 3153a2c98d7201b1ae8104c346db58e19f322cc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:53:54 +0600
Subject: [PATCH 0790/1515] [tvigle] Skip tests

---
 youtube_dl/extractor/tvigle.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index aa07c8251..dc3a8334a 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -30,6 +30,7 @@ class TvigleIE(InfoExtractor):
                 'duration': 6586,
                 'age_limit': 12,
             },
+            'skip': 'georestricted',
         },
         {
             'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
@@ -42,6 +43,7 @@ class TvigleIE(InfoExtractor):
                 'duration': 186.080,
                 'age_limit': 0,
             },
+            'skip': 'georestricted',
         }, {
             'url': 'https://cloud.tvigle.ru/video/5267604/',
             'only_matching': True,

From 6800d3372f35e08dcc4d34d06601815bf0cb0a3d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 23:10:18 +0800
Subject: [PATCH 0791/1515] [YoutubeDL] Support DASH manifest downloading

---
 youtube_dl/downloader/dash.py   | 50 +++++++++++++++++++++++++++++++++
 youtube_dl/downloader/http.py   |  4 +++
 youtube_dl/extractor/youtube.py |  6 ++++
 3 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/downloader/dash.py

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
new file mode 100644
index 000000000..18eca2c04
--- /dev/null
+++ b/youtube_dl/downloader/dash.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
+import re
+
+
+class DashSegmentsFD(FileDownloader):
+    """
+    Download segments in a DASH manifest
+    """
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+        base_url = info_dict['url']
+        segment_urls = info_dict['segment_urls']
+
+        self.byte_counter = 0
+
+        def append_url_to_file(outf, target_url, target_name):
+            self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
+            req = compat_urllib_request.Request(target_url)
+            data = self.ydl.urlopen(req).read()
+            outf.write(data)
+            self.byte_counter += len(data)
+
+        def combine_url(base_url, target_url):
+            if re.match(r'^https?://', target_url):
+                return target_url
+            return '%s/%s' % (base_url, target_url)
+
+        with open(tmpfilename, 'wb') as outf:
+            append_url_to_file(
+                outf, combine_url(base_url, info_dict['initialization_url']),
+                'initialization segment')
+            for i, segment_url in enumerate(segment_urls):
+                append_url_to_file(
+                    outf, combine_url(base_url, segment_url),
+                    'segment %d / %d' % (i + 1, len(segment_urls)))
+
+        self.try_rename(tmpfilename, filename)
+
+        self._hook_progress({
+            'downloaded_bytes': self.byte_counter,
+            'total_bytes': self.byte_counter,
+            'filename': filename,
+            'status': 'finished',
+        })
+
+        return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b7f144af9..ceacb8522 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -6,6 +6,7 @@ import socket
 import time
 
 from .common import FileDownloader
+from .dash import DashSegmentsFD
 from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
@@ -19,6 +20,9 @@ from ..utils import (
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
+        if info_dict.get('initialization_url') and list(filter(None, info_dict.get('segment_urls', []))):
+            return DashSegmentsFD(self.ydl, self.params).real_download(filename, info_dict)
+
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
         stream = None
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aacb999ce..5d1297e0d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -802,6 +802,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # TODO implement WebVTT downloading
                     pass
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
                     format_id = r.attrib['id']
                     video_url = url_el.text
                     filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
@@ -815,6 +816,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
+                    if segment_list:
+                        f.update({
+                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')]
+                        })
                     try:
                         existing_format = next(
                             fo for fo in formats

From e4ac7bb1e598b0317742737ea06c162fa7f22cd4 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Wed, 3 Jun 2015 19:18:41 +0300
Subject: [PATCH 0792/1515] [nova] Revert "Fix extension extraction bug"

This reverts commit 9464a194dbf48989c486fa2de9e1aebc59e28ed4.
---
 youtube_dl/extractor/nova.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 85253b6ed..3f9c776ef 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
-    determine_ext,
     unified_strdate,
 )
 
@@ -44,7 +43,7 @@ class NovaIE(InfoExtractor):
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
             'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
             'thumbnail': 're:^https?://.*\.(?:jpg)',
@@ -57,7 +56,7 @@ class NovaIE(InfoExtractor):
         'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
         'info_dict': {
             'id': '1756858',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Televizní noviny - 30. 5. 2015',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
             'upload_date': '20150530',
@@ -140,8 +139,21 @@ class NovaIE(InfoExtractor):
 
         mediafile = config['mediafile']
         video_url = mediafile['src']
-        ext = determine_ext(video_url)
-        video_url = video_url.replace('&%s:' % ext, '')
+
+        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
+        if m:
+            formats = [{
+                'url': m.group('url'),
+                'app': m.group('app'),
+                'play_path': m.group('playpath'),
+                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
+                'ext': 'flv',
+            }]
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+        self._sort_formats(formats)
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = clean_html(self._og_search_description(webpage, default=None))
@@ -163,6 +175,5 @@ class NovaIE(InfoExtractor):
             'description': description,
             'upload_date': upload_date,
             'thumbnail': thumbnail,
-            'url': video_url,
-            'ext': ext,
+            'formats': formats,
         }

From 4c8fea92f350d5a3f33d505980ac750b05a9cd34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Jun 2015 23:50:38 +0200
Subject: [PATCH 0793/1515] [test/aes] Fix on python 3.3 and higher

Since 878563c847fa5248eedbd44187536dec04643eaf the aes functions only accepts the base64 data as a unicode string.
---
 test/test_aes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_aes.py b/test/test_aes.py
index 4dc7de7b5..315a3f5ae 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -39,7 +39,7 @@ class TestAES(unittest.TestCase):
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8]) +
             b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
-        )
+        ).decode('utf-8')
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
@@ -47,7 +47,7 @@ class TestAES(unittest.TestCase):
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8]) +
             b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
-        )
+        ).decode('utf-8')
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 

From eedda32e6bc4620d704eabab7a2c8f4b1f1a9169 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 11:27:18 +0800
Subject: [PATCH 0794/1515] [qqmusic] Fix toplist

---
 youtube_dl/extractor/qqmusic.py | 58 +++++++++++++--------------------
 1 file changed, 23 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index b540033e2..48f28ffe9 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -181,60 +181,48 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=toplist&p=global_12',
+        'url': 'http://y.qq.com/#type=toplist&p=global_123',
         'info_dict': {
-            'id': 'global_12',
-            'title': 'itunes榜',
+            'id': 'global_123',
+            'title': '美国iTunes榜',
         },
         'playlist_count': 10,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=top_6',
+        'url': 'http://y.qq.com/#type=toplist&p=top_3',
         'info_dict': {
-            'id': 'top_6',
+            'id': 'top_3',
             'title': 'QQ音乐巅峰榜·欧美',
+            'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
+                           '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
+                           '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'
+                           '登录用户完整播放一首歌曲，记为一次有效播放；同一用户收听同一首歌曲，每天记录为1次有效播放'
         },
         'playlist_count': 100,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=global_5',
+        'url': 'http://y.qq.com/#type=toplist&p=global_106',
         'info_dict': {
-            'id': 'global_5',
-            'title': '韩国mnet排行榜',
+            'id': 'global_106',
+            'title': '韩国Mnet榜',
         },
         'playlist_count': 50,
     }]
 
-    @staticmethod
-    def strip_qq_jsonp(code):
-        return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
         list_type, num_id = list_id.split("_")
 
-        list_page = self._download_webpage(
-            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
+        toplist_json = self._download_json(
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg?type=%s&topid=%s&format=json'
+            % (list_type, num_id),
             list_id, 'Download toplist page')
 
-        entries = []
-        if list_type == 'top':
-            jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
-        else:
-            jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['data']['songmid'], 'QQMusic', song['data']['songmid']
+            ) for song in toplist_json['songlist']
+        ]
 
-        toplist_json = self._download_json(
-            jsonp_url, list_id, note='Retrieve toplist json',
-            errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
-
-        for song in toplist_json['l']:
-            s = song['s']
-            song_mid = s.split("|")[20]
-            entries.append(self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
-                song_mid))
-
-        list_name = self._html_search_regex(
-            r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
-            default=None)
-
-        return self.playlist_result(entries, list_id, list_name)
+        list_name = toplist_json['topinfo']['ListName']
+        list_description = toplist_json['topinfo']['info']
+        return self.playlist_result(entries, list_id, list_name, list_description)

From ed15e9ba02382fc7db22e6176068d2220c00a32e Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 17:32:06 +0800
Subject: [PATCH 0795/1515] [qqmusic] Remove unused import

---
 youtube_dl/extractor/qqmusic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 48f28ffe9..9943fcddb 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,7 +9,6 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
-    js_to_json,
 )
 from ..compat import compat_urllib_request
 

From 55e5841f14131ab61359535fdcc44e1564d555b8 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 17:41:29 +0800
Subject: [PATCH 0796/1515] [qqmusic] Extract additional formats (mp3-128,
 mp3-320)

---
 youtube_dl/extractor/qqmusic.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index b540033e2..4b4ef4993 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -19,10 +19,10 @@ class QQMusicIE(InfoExtractor):
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
-        'md5': 'bed90b6db2a7a7a7e11bc585f471f63a',
+        'md5': '9ce1c1c8445f561506d2e3cfb0255705',
         'info_dict': {
             'id': '004295Et37taLD',
-            'ext': 'm4a',
+            'ext': 'mp3',
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
@@ -30,6 +30,12 @@ class QQMusicIE(InfoExtractor):
         }
     }]
 
+    _FORMATS = {
+        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40},
+        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30},
+        'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
+    }
+
     # Reference: m_r_GetRUin() in top_player.js
     # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
     @staticmethod
@@ -69,11 +75,19 @@ class QQMusicIE(InfoExtractor):
             'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
             mid, note='Retrieve vkey', errnote='Unable to get vkey',
             transform_source=strip_jsonp)['key']
-        song_url = 'http://cc.stream.qqmusic.qq.com/C200%s.m4a?vkey=%s&guid=%s&fromtag=0' % (mid, vkey, guid)
+
+        formats = []
+        for k, sf in self._FORMATS.items():
+            formats.append({
+                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
+                       % (sf['prefix'], mid, sf['ext'], vkey, guid),
+                'format': k, 'format_id': k, 'preference': sf['preference']
+            })
+        self._sort_formats(formats)
 
         return {
             'id': mid,
-            'url': song_url,
+            'formats': formats,
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,

From b9258c61789388b49792ebdceb5d804217a36da5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:05:33 +0800
Subject: [PATCH 0797/1515] [YoutubeDL] Change how DashSegmentsFD is selected

---
 youtube_dl/downloader/__init__.py | 2 ++
 youtube_dl/downloader/http.py     | 4 ----
 youtube_dl/extractor/youtube.py   | 3 ++-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index f110830c4..1b618ab54 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -8,6 +8,7 @@ from .hls import NativeHlsFD
 from .http import HttpFD
 from .rtsp import RtspFD
 from .rtmp import RtmpFD
+from .dash import DashSegmentsFD
 
 from ..utils import (
     determine_protocol,
@@ -20,6 +21,7 @@ PROTOCOL_MAP = {
     'mms': RtspFD,
     'rtsp': RtspFD,
     'f4m': F4mFD,
+    'dash_segments': DashSegmentsFD,
 }
 
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index ceacb8522..b7f144af9 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -6,7 +6,6 @@ import socket
 import time
 
 from .common import FileDownloader
-from .dash import DashSegmentsFD
 from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
@@ -20,9 +19,6 @@ from ..utils import (
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
-        if info_dict.get('initialization_url') and list(filter(None, info_dict.get('segment_urls', []))):
-            return DashSegmentsFD(self.ydl, self.params).real_download(filename, info_dict)
-
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
         stream = None
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5d1297e0d..692d4d8db 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -819,7 +819,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if segment_list:
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')]
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'protocol': 'dash_segments',
                         })
                     try:
                         existing_format = next(

From 453a1617aac6e8000ed947cad7d88817c5740ede Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:12:05 +0800
Subject: [PATCH 0798/1515] [downloader/dash] Reorder imports

---
 youtube_dl/downloader/dash.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 18eca2c04..5f14658ba 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
-from .common import FileDownloader
-from ..compat import compat_urllib_request
 
 import re
 
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
 
 class DashSegmentsFD(FileDownloader):
     """

From 423d2be5f8c5e70d202ddfa63f3e5365e6afe823 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:27:29 +0800
Subject: [PATCH 0799/1515] [downloader/dash] Rename the protocol

'http_dash_segments' looks more like a protocol name than
'dash_segments'
---
 youtube_dl/downloader/__init__.py | 2 +-
 youtube_dl/extractor/youtube.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 1b618ab54..dccc59212 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -21,7 +21,7 @@ PROTOCOL_MAP = {
     'mms': RtspFD,
     'rtsp': RtspFD,
     'f4m': F4mFD,
-    'dash_segments': DashSegmentsFD,
+    'http_dash_segments': DashSegmentsFD,
 }
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 692d4d8db..6d288e848 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -820,7 +820,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
-                            'protocol': 'dash_segments',
+                            'protocol': 'http_dash_segments',
                         })
                     try:
                         existing_format = next(

From 56c837ccb75b639d362397095f33300229c4bd1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:34:48 +0600
Subject: [PATCH 0800/1515] [tnaflix] Fix typo

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 59af9aba0..bc51bae37 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -33,7 +33,7 @@ class TNAFlixIE(InfoExtractor):
         },
         {
             'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-            'matching_only': True,
+            'only_matching': True,
         }
     ]
 

From e52c0bd0eb853d4d242872e1d9ff5426a35dd30c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:37:05 +0600
Subject: [PATCH 0801/1515] [tnaflix] Modernize

---
 youtube_dl/extractor/tnaflix.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index bc51bae37..3e335d653 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -51,9 +51,8 @@ class TNAFlixIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        duration = self._html_search_meta('duration', webpage, 'duration', default=None)
-        if duration:
-            duration = parse_duration(duration[1:])
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', default=None))
 
         cfg_url = self._proto_relative_url(self._html_search_regex(
             self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')

From 3ce9bc712acd88df8499dd0982277c8f64b0d15a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:39:03 +0600
Subject: [PATCH 0802/1515] [empflix] Fix typo

---
 youtube_dl/extractor/empflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 9a5a8f4bb..4827022e0 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -26,6 +26,6 @@ class EMPFlixIE(TNAFlixIE):
         },
         {
             'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
-            'matching_only': True,
+            'only_matching': True,
         }
     ]

From 3d6388e34ea41d937f39e561b7731f1389971a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:42:37 +0600
Subject: [PATCH 0803/1515] [tnaflix] Fix relative URLs (empflix)

---
 youtube_dl/extractor/tnaflix.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 3e335d653..c282865b2 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -61,14 +61,15 @@ class TNAFlixIE(InfoExtractor):
             cfg_url, display_id, note='Downloading metadata',
             transform_source=fix_xml_ampersands)
 
-        thumbnail = cfg_xml.find('./startThumb').text
+        thumbnail = self._proto_relative_url(
+            cfg_xml.find('./startThumb').text, 'http:')
 
         formats = []
         for item in cfg_xml.findall('./quality/item'):
             video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
             format_id = item.find('res').text
             fmt = {
-                'url': video_url,
+                'url': self._proto_relative_url(video_url, 'http:'),
                 'format_id': format_id,
             }
             m = re.search(r'^(\d+)', format_id)

From 9d4f213f90d6024c7748f4defdc7b45f2351b0da Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 00:52:18 +0800
Subject: [PATCH 0804/1515] [qqmusic:toplist] List name and description are
 optional

---
 youtube_dl/extractor/qqmusic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 9943fcddb..f773332a8 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -222,6 +222,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             ) for song in toplist_json['songlist']
         ]
 
-        list_name = toplist_json['topinfo']['ListName']
-        list_description = toplist_json['topinfo']['info']
+        topinfo = toplist_json.get('topinfo', {})
+        list_name = topinfo.get('ListName')
+        list_description = topinfo.get('info')
         return self.playlist_result(entries, list_id, list_name, list_description)

From f5c78d118ba2d7e5e4a1ccd40c97fc1bf85a8dcf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Jun 2015 21:49:02 +0200
Subject: [PATCH 0805/1515] release 2015.06.04

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a421ae62b..d147b53fe 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -10,6 +10,7 @@
  - **56.com**
  - **5min**
  - **8tracks**
+ - **91porn**
  - **9gag**
  - **abc.net.au**
  - **Abc7News**
@@ -319,6 +320,7 @@
  - **Noco**
  - **Normalboots**
  - **NosVideo**
+ - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **novamov**: NovaMov
  - **Nowness**
  - **NowTV**
@@ -431,6 +433,8 @@
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
  - **Sohu**
+ - **soompi**
+ - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
  - **soundcloud:set**
@@ -505,6 +509,7 @@
  - **Trilulilu**
  - **TruTube**
  - **Tube8**
+ - **TubiTv**
  - **Tudou**
  - **Tumblr**
  - **TuneIn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 653710131..84224b7a7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.29'
+__version__ = '2015.06.04'

From 0e805e782bd05951ca3f420cf2a050e2ac3ae846 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Jun 2015 21:54:33 +0200
Subject: [PATCH 0806/1515] release 2015.06.04.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 84224b7a7..9cf84ff71 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.04'
+__version__ = '2015.06.04.1'

From 8b8cde21406b53f5aeb6586dab03a9d78d62e631 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 5 Jun 2015 06:04:26 +0800
Subject: [PATCH 0807/1515] [qqmusic] Set abr for mp3 formats

---
 youtube_dl/extractor/qqmusic.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 4b4ef4993..dc300e189 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -31,8 +31,8 @@ class QQMusicIE(InfoExtractor):
     }]
 
     _FORMATS = {
-        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40},
-        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30},
+        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40, 'abr': 320},
+        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30, 'abr': 128},
         'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
     }
 
@@ -77,11 +77,12 @@ class QQMusicIE(InfoExtractor):
             transform_source=strip_jsonp)['key']
 
         formats = []
-        for k, sf in self._FORMATS.items():
+        for k, f in self._FORMATS.items():
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (sf['prefix'], mid, sf['ext'], vkey, guid),
-                'format': k, 'format_id': k, 'preference': sf['preference']
+                       % (f['prefix'], mid, f['ext'], vkey, guid),
+                'format': k, 'format_id': k, 'preference': f['preference'],
+                'abr': f.get('abr')
             })
         self._sort_formats(formats)
 

From d31573fa37c8db7133492baf0a6be3ece643f8ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 22:55:29 +0800
Subject: [PATCH 0808/1515] [teamcoco] Handle incomplete m3u8 URLs (fixes
 #5798)

There are 2 TODOs. I don't know how to handle these cases correctly.
---
 youtube_dl/extractor/teamcoco.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index b2a4b1fc0..d1b7264b4 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -51,6 +51,17 @@ class TeamcocoIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # m3u8 downloads
             }
+        }, {
+            'url': 'http://teamcoco.com/video/full-episode-mon-6-1-joel-mchale-jake-tapper-and-musical-guest-courtney-barnett?playlist=x;eyJ0eXBlIjoidGFnIiwiaWQiOjl9',
+            'info_dict': {
+                'id': '89341',
+                'ext': 'mp4',
+                'title': 'Full Episode - Mon. 6/1 - Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
+                'description': 'Guests: Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 downloads
+            }
         }
     ]
     _VIDEO_ID_REGEXES = (
@@ -110,9 +121,23 @@ class TeamcocoIE(InfoExtractor):
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
             if determine_ext(filed['url']) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    filed['url'], video_id, ext='mp4'))
+                # compat_urllib_parse.urljoin does not work here
+                if filed['url'].startswith('/'):
+                    m3u8_url = 'http://ht.cdn.turner.com/tbs/big/teamcoco' + filed['url']
+                else:
+                    m3u8_url = filed['url']
+                m3u8_formats = self._extract_m3u8_formats(
+                    m3u8_url, video_id, ext='mp4')
+                for m3u8_format in m3u8_formats:
+                    if m3u8_format not in formats:
+                        formats.append(m3u8_format)
+            elif determine_ext(filed['url']) == 'f4m':
+                # TODO Correct f4m extraction
+                continue
             else:
+                if filed['url'].startswith('/mp4:protected/'):
+                    # TODO Correct extraction for these files
+                    continue
                 m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
                 if m_format is not None:
                     format_id = m_format.group(1)

From f00a650705a5e5b4f2b540ea8133a1752e63dd81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 23:16:34 +0800
Subject: [PATCH 0809/1515] [qqmusic] Rearrange codes

---
 youtube_dl/extractor/qqmusic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e24ddaefe..c903bee58 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -80,7 +80,9 @@ class QQMusicIE(InfoExtractor):
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
                        % (f['prefix'], mid, f['ext'], vkey, guid),
-                'format': k, 'format_id': k, 'preference': f['preference'],
+                'format': k,
+                'format_id': k,
+                'preference': f['preference'],
                 'abr': f.get('abr')
             })
         self._sort_formats(formats)

From e8ac61e840b5c02e406b910f5f0eed3d8b331969 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 23:19:25 +0800
Subject: [PATCH 0810/1515] [qqmusic] Use meaningful variable names

---
 youtube_dl/extractor/qqmusic.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c903bee58..bafa81c21 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -76,14 +76,14 @@ class QQMusicIE(InfoExtractor):
             transform_source=strip_jsonp)['key']
 
         formats = []
-        for k, f in self._FORMATS.items():
+        for format_id, details in self._FORMATS.items():
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (f['prefix'], mid, f['ext'], vkey, guid),
-                'format': k,
-                'format_id': k,
-                'preference': f['preference'],
-                'abr': f.get('abr')
+                       % (details['prefix'], mid, details['ext'], vkey, guid),
+                'format': format_id,
+                'format_id': format_id,
+                'preference': details['preference'],
+                'abr': details.get('abr'),
             })
         self._sort_formats(formats)
 

From dfe7dd9bdb45ec765c9b335c149e9913cf7e413f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 02:54:57 +0300
Subject: [PATCH 0811/1515] [izlesene] Unquote video URLs and simplify

---
 youtube_dl/extractor/izlesene.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 99a1361f8..753cb98ea 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -67,9 +68,9 @@ class IzleseneIE(InfoExtractor):
 
         uploader = self._html_search_regex(
             r"adduserUsername\s*=\s*'([^']+)';",
-            webpage, 'uploader', fatal=False, default='')
+            webpage, 'uploader', fatal=False)
         timestamp = parse_iso8601(self._html_search_meta(
-            'uploadDate', webpage, 'upload date', fatal=False))
+            'uploadDate', webpage, 'upload date'))
 
         duration = float_or_none(self._html_search_regex(
             r'"videoduration"\s*:\s*"([^"]+)"',
@@ -86,8 +87,7 @@ class IzleseneIE(InfoExtractor):
 
         # Might be empty for some videos.
         streams = self._html_search_regex(
-            r'"qualitylevel"\s*:\s*"([^"]+)"',
-            webpage, 'streams', fatal=False, default='')
+            r'"qualitylevel"\s*:\s*"([^"]+)"', webpage, 'streams', default='')
 
         formats = []
         if streams:
@@ -95,15 +95,15 @@ class IzleseneIE(InfoExtractor):
                 quality, url = re.search(r'\[(\w+)\](.+)', stream).groups()
                 formats.append({
                     'format_id': '%sp' % quality if quality else 'sd',
-                    'url': url,
+                    'url': compat_urllib_parse_unquote(url),
                     'ext': ext,
                 })
         else:
             stream_url = self._search_regex(
-                r'"streamurl"\s?:\s?"([^"]+)"', webpage, 'stream URL')
+                r'"streamurl"\s*:\s*"([^"]+)"', webpage, 'stream URL')
             formats.append({
                 'format_id': 'sd',
-                'url': stream_url,
+                'url': compat_urllib_parse_unquote(stream_url),
                 'ext': ext,
             })
 

From c33c547d66b20064f83932cdaa04823b17a96b70 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 02:57:12 +0300
Subject: [PATCH 0812/1515] [izlesene] Avoid timestamp differences in tests due
 to DST

---
 youtube_dl/extractor/izlesene.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 753cb98ea..bc226fa67 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -31,7 +31,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'pelikzzle',
-                'timestamp': 1404302298,
+                'timestamp': int,
                 'upload_date': '20140702',
                 'duration': 95.395,
                 'age_limit': 0,
@@ -47,7 +47,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'Tarkan Dortmund 2006 Konseri',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'parlayankiz',
-                'timestamp': 1163322193,
+                'timestamp': int,
                 'upload_date': '20061112',
                 'duration': 253.666,
                 'age_limit': 0,

From 54eb81a087516e9d040bc1ad274c0a64b51dd1d1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 03:11:43 +0300
Subject: [PATCH 0813/1515] [pornovoisines] Improve average_rating extraction
 and update test case

---
 youtube_dl/extractor/pornovoisines.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index 9688ed948..eba4dfbb3 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -34,7 +34,7 @@ class PornoVoisinesIE(InfoExtractor):
             'duration': 120,
             'view_count': int,
             'average_rating': float,
-            'categories': ['Débutante', 'Scénario', 'Sodomie'],
+            'categories': ['Débutantes', 'Scénario', 'Sodomie'],
             'age_limit': 18,
         }
     }
@@ -71,7 +71,7 @@ class PornoVoisinesIE(InfoExtractor):
         view_count = int_or_none(self._search_regex(
             r'(\d+) vues', webpage, 'view count', fatal=False))
         average_rating = self._search_regex(
-            r'Note : (\d+,\d+)', webpage, 'average rating', fatal=False)
+            r'Note\s*:\s*(\d+(?:,\d+)?)', webpage, 'average rating', fatal=False)
         if average_rating:
             average_rating = float_or_none(average_rating.replace(',', '.'))
 

From 3d8e9573a470594df6fa471dc33c4c4b938b668a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jun 2015 06:25:37 +0600
Subject: [PATCH 0814/1515] [youtube:channel] Improve channel id extraction
 (#5904)

---
 youtube_dl/extractor/youtube.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aacb999ce..419f7b019 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1406,10 +1406,12 @@ class YoutubeChannelIE(InfoExtractor):
         channel_page = self._download_webpage(
             url + '?view=57', channel_id,
             'Downloading channel page', fatal=False)
-        channel_playlist_id = self._search_regex(
-            [r'<meta itemprop="channelId" content="([^"]+)">',
-             r'data-channel-external-id="([^"]+)"'],
-            channel_page, 'channel id', default=None)
+        channel_playlist_id = self._html_search_meta(
+            'channelId', channel_page, 'channel id', default=None)
+        if not channel_playlist_id:
+            channel_playlist_id = self._search_regex(
+                r'data-channel-external-id="([^"]+)"',
+                channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
             return self.url_result(

From 223544552fcfec0c5c6a83326520c614e4489cbb Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sat, 9 May 2015 03:53:43 +0300
Subject: [PATCH 0815/1515] [Ruutu] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ruutu.py    | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/ruutu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8ec0c1032..860023d14 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -448,6 +448,7 @@ from .rutube import (
     RutubePersonIE,
 )
 from .rutv import RUTVIE
+from .ruutu import RuutuIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
new file mode 100644
index 000000000..e346434f9
--- /dev/null
+++ b/youtube_dl/extractor/ruutu.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
+import re
+
+
+class RuutuIE(InfoExtractor):
+    _VALID_URL = r'http://(www\.)?ruutu\.fi/ohjelmat/(?:[^/]+/)?(?P<id>.*)$'
+    _TESTS = [
+        {
+            'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+            'md5': 'ab2093f39be1ca8581963451b3c0234f',
+            'info_dict': {
+                'id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+                'ext': 'mp4',
+                'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
+                'description': 'Toinen toistaan huikeampia ohjelmaideoita ja täysin päätöntä sekoilua? No sitä juuri nimenomaan. Metro Helsingin Iltapäivän vieraaksi saapui Tuomas Kauhanen ja he Petra Kalliomaan kanssa keskustelivat hieman ennen lähetyksen alkua, mutta kamerat olivatkin jo päällä.',
+            },
+            'params': {
+                'format': 'http-1000',
+            }
+        },
+        {
+            'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
+            'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
+            'info_dict': {
+                'id': 'superpesis-katso-koko-kausi-ruudussa',
+                'ext': 'mp4',
+                'title': 'Superpesis: katso koko kausi Ruudussa',
+                'description': 'Huippujännittävän Superpesiksen suoria ottelulähetyksiä seurataan Ruudussa kauden alusta viimeiseen finaaliin asti. Katso lisätiedot osoitteesta ruutu.fi/superpesis.',
+            },
+            'params': {
+                'format': 'http-1000',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id)
+        media_id = self._html_search_regex(r'data-media-id="(\d+)"', webpage, 'media_id')
+        media_json = self._parse_json(self._search_regex(r'jQuery.extend\([^,]+, (.*)\);', webpage, 'media_data'), video_id)
+        xml_url = media_json['ruutuplayer']['xmlUrl'].replace('{ID}', media_id)
+        media_xml = self._download_xml(xml_url, media_id)
+
+        formats = []
+        parsed_urls = []
+        for fmt in media_xml.findall('.//Clip//'):
+            url = fmt.text
+            if not fmt.tag.endswith('File') or url in parsed_urls or \
+                    'NOT_USED' in url:
+                continue
+
+            if url.endswith('m3u8'):
+                formats.extend(self._extract_m3u8_formats(url, media_id, m3u8_id='hls'))
+                parsed_urls.append(url)
+            elif url.endswith('f4m'):
+                formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
+                parsed_urls.append(url)
+            else:
+                proto = compat_urllib_parse_urlparse(url).scheme
+                width_str, height_str = fmt.get('resolution').split('x')
+                tbr = int(fmt.get('bitrate', 0))
+                formats.append({
+                    'format_id': '%s-%d' % (proto, tbr),
+                    'url': url,
+                    'width': int(width_str),
+                    'height': int(height_str),
+                    'tbr': tbr,
+                    'ext': url.rsplit('.', 1)[-1],
+                    'live': True,
+                    'protocol': proto,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int(media_xml.find('.//Runtime').text),
+            'age_limit': int(media_xml.find('.//AgeLimit').text),
+        }

From a9e58ecd3fdfa93cdc8a7f9fc852dbbd0814d6a4 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 13:56:46 +0300
Subject: [PATCH 0816/1515] [turbo] Improve description extraction

`og:description` is empty for some videos.
---
 youtube_dl/extractor/turbo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/turbo.py b/youtube_dl/extractor/turbo.py
index 29703a8a9..7ae63a499 100644
--- a/youtube_dl/extractor/turbo.py
+++ b/youtube_dl/extractor/turbo.py
@@ -23,7 +23,7 @@ class TurboIE(InfoExtractor):
             'ext': 'mp4',
             'duration': 3715,
             'title': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
-            'description': 'Retrouvez dans cette rubrique toutes les vidéos de l\'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
+            'description': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -42,7 +42,7 @@ class TurboIE(InfoExtractor):
         title = xpath_text(item, './title', 'title')
         duration = int_or_none(xpath_text(item, './durate', 'duration'))
         thumbnail = xpath_text(item, './visuel_clip', 'thumbnail')
-        description = self._og_search_description(webpage)
+        description = self._html_search_meta('description', webpage)
 
         formats = []
         get_quality = qualities(['3g', 'sd', 'hq'])

From 05aa9c82d90644af406519e5e25fefb0884d504e Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 13:58:20 +0300
Subject: [PATCH 0817/1515] [sunporno] Fix view_count extraction

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 854d01bee..e527aa971 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -44,7 +44,7 @@ class SunPornoIE(InfoExtractor):
             webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'class="views">\s*(\d+)\s*<',
+            r'class="views">(?:<noscript>)?\s*(\d+)\s*<',
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'(\d+)</b> Comments?',

From 4da31bd56629054497634d041035e4bd6fcfacbb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 Jun 2015 22:22:26 +0800
Subject: [PATCH 0818/1515] [youtube] Fix a FutureWarning from
 xml.etree.ElementTree

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d288e848..2424ac2c0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -816,7 +816,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
-                    if segment_list:
+                    if len(segment_list):
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],

From f1da861018924e6f442ffedd9a5682055c79aea6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:37:29 +0800
Subject: [PATCH 0819/1515] [iqiyi] PEP8

---
 youtube_dl/extractor/iqiyi.py | 56 +++++++++++++++++------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 747f3f902..597441baf 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,19 +16,20 @@ import random
 import zlib
 import hashlib
 
+
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
 
     _TEST = {
-            'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-            'md5': '2cb594dc2781e6c941a110d8f358118b',
-            'info_dict': {
-                'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
-                'title': '美国德州空中惊现奇异云团 酷似UFO',
-                'ext': 'f4v',
-            }
+        'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
+        'md5': '2cb594dc2781e6c941a110d8f358118b',
+        'info_dict': {
+            'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+            'title': '美国德州空中惊现奇异云团 酷似UFO',
+            'ext': 'f4v',
+        }
     }
 
     def construct_video_urls(self, data, video_id, _uuid, bid):
@@ -46,7 +47,7 @@ class IqiyiIE(InfoExtractor):
             c = len(b)
             s = ''
             for i in range(c - 1, -1, -1):
-                a = do_xor(int(b[c-i-1], 16), i)
+                a = do_xor(int(b[c - i - 1], 16), i)
                 s += chr(a)
             return s[::-1]
 
@@ -54,15 +55,14 @@ class IqiyiIE(InfoExtractor):
             mg = ')(*&^flash@#$%a'
             tm = self._download_json(
                 'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
-            t = str(int(math.floor(int(tm)/(600.0))))
-            return hashlib.md5(
-                 (t+mg+x).encode('utf8')).hexdigest()
+            t = str(int(math.floor(int(tm) / (600.0))))
+            return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
         # get accept format
         # getting all format will spend minutes for a big video.
         if bid == 'best':
-            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs'] \
-                   if 0 < int(i['bid']) <= 10]
+            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs']
+                    if 0 < int(i['bid']) <= 10]
             bid = str(max(bids))
 
         video_urls_dict = {}
@@ -117,24 +117,24 @@ class IqiyiIE(InfoExtractor):
 
     def get_format(self, bid):
         _dict = {
-            '1'  : 'h6',
-            '2'  : 'h5',
-            '3'  : 'h4',
-            '4'  : 'h3',
-            '5'  : 'h2',
-            '10' : 'h1'
+            '1': 'h6',
+            '2': 'h5',
+            '3': 'h4',
+            '4': 'h3',
+            '5': 'h2',
+            '10': 'h1'
         }
         return _dict.get(str(bid), None)
 
     def get_bid(self, format_id):
         _dict = {
-            'h6'   : '1',
-            'h5'   : '2',
-            'h4'   : '3',
-            'h3'   : '4',
-            'h2'   : '5',
-            'h1'   : '10',
-            'best' : 'best'
+            'h6': '1',
+            'h5': '2',
+            'h4': '3',
+            'h3': '4',
+            'h2': '5',
+            'h1': '10',
+            'best': 'best'
         }
         return _dict.get(format_id, None)
 
@@ -207,7 +207,7 @@ class IqiyiIE(InfoExtractor):
         for format_id in video_urls_dict:
             video_urls = video_urls_dict[format_id]
             for i, video_url_info in enumerate(video_urls):
-                if len(entries) < i+1:
+                if len(entries) < i + 1:
                     entries.append({'formats': []})
                 entries[i]['formats'].append(
                     {
@@ -222,7 +222,7 @@ class IqiyiIE(InfoExtractor):
             self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
-                    'id': '_part%d' % (i+1),
+                    'id': '_part%d' % (i + 1),
                     'title': title,
                 }
             )

From 7012620e2b9355d25ddfc855fc5990af938f04d8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:44:54 +0800
Subject: [PATCH 0820/1515] [iqiyi] Remove format selection codes

---
 youtube_dl/extractor/iqiyi.py | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 597441baf..5645fb6ee 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -32,7 +32,7 @@ class IqiyiIE(InfoExtractor):
         }
     }
 
-    def construct_video_urls(self, data, video_id, _uuid, bid):
+    def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -58,13 +58,6 @@ class IqiyiIE(InfoExtractor):
             t = str(int(math.floor(int(tm) / (600.0))))
             return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
-        # get accept format
-        # getting all format will spend minutes for a big video.
-        if bid == 'best':
-            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs']
-                    if 0 < int(i['bid']) <= 10]
-            bid = str(max(bids))
-
         video_urls_dict = {}
         for i in data['vp']['tkl'][0]['vs']:
             if 0 < int(i['bid']) <= 10:
@@ -80,12 +73,6 @@ class IqiyiIE(InfoExtractor):
                 if t.endswith('mp4'):
                     video_urls_info = i['flvs']
 
-            if int(i['bid']) != int(bid):  # ignore missing match format
-                video_urls.extend(
-                    [('http://example.com/v.flv', ii['b']) for ii in video_urls_info])
-                video_urls_dict[format_id] = video_urls
-                continue
-
             for ii in video_urls_info:
                 vl = ii['l']
                 if not vl.startswith('/'):
@@ -193,14 +180,9 @@ class IqiyiIE(InfoExtractor):
 
         title = data['vi']['vn']
 
-        format = self._downloader.params.get('format', None)
-        bid = self.get_bid(format) if format else 'best'
-        if not bid:
-            raise ExtractorError('Can\'t get format.')
-
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(
-            data, video_id, _uuid, bid)
+            data, video_id, _uuid)
 
         # construct info
         entries = []

From 29e7e0781b1b8e276c28a079bc5b18e1b0db2d5e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:56:08 +0800
Subject: [PATCH 0821/1515] [iqiyi] Simplify and improve regex patterns

See the comments in #5849
---
 youtube_dl/extractor/iqiyi.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 5645fb6ee..18a7587a2 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -161,12 +161,11 @@ class IqiyiIE(InfoExtractor):
         webpage = self._download_webpage(
             url, 'temp_id', note='download video page')
         tvid = self._search_regex(
-            r'tvId ?= ?(\'|\")(?P<tvid>\d+)', webpage, 'tvid', flags=re.I, group='tvid')
+            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(
-            r'videoId ?= ?(\'|\")(?P<video_id>[a-z\d]+)',
-            webpage, 'video_id', flags=re.I, group='video_id')
+            r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
         swf_url = self._search_regex(
-            r'(?P<swf>http://.+?MainPlayer.+?\.swf)', webpage, 'swf')
+            r'(http://.+?MainPlayer.+?\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
         enc_key = self.get_enc_key(swf_url, video_id)

From aacda28b28c1804866d634c5c5086b3d53cb2b2f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:32:03 +0800
Subject: [PATCH 0822/1515] [iqiyi] Give error message for assertion failures

---
 youtube_dl/extractor/iqiyi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 18a7587a2..dc35c3380 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -171,7 +171,10 @@ class IqiyiIE(InfoExtractor):
         enc_key = self.get_enc_key(swf_url, video_id)
 
         raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
-        assert raw_data['code'] == 'A000000'
+
+        if raw_data['code'] != 'A000000':
+            raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
+
         if not raw_data['data']['vp']['tkl']:
             raise ExtractorError('No support iQiqy VIP video')
 

From 958d0b659b80d4493d045d4da82074ed68ed6c4e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:35:09 +0800
Subject: [PATCH 0823/1515] [iqiyi] Reorder imports

---
 youtube_dl/extractor/iqiyi.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index dc35c3380..36029361a 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -1,20 +1,17 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-
-from ..compat import compat_urllib_parse
-
-from ..utils import ExtractorError
-
+import hashlib
+import math
+import random
 import re
 import time
 import uuid
-import math
-import random
 import zlib
-import hashlib
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import ExtractorError
 
 
 class IqiyiIE(InfoExtractor):

From ffba4edb067238b593b98c71f4293e9b60ba95ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:52:51 +0800
Subject: [PATCH 0824/1515] [iqiyi] Improve some variable names and add
 download notes

---
 youtube_dl/extractor/iqiyi.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 36029361a..c17e1fde4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -48,35 +48,37 @@ class IqiyiIE(InfoExtractor):
                 s += chr(a)
             return s[::-1]
 
-        def get_path_key(x):
+        def get_path_key(x, format_id, segment_index):
             mg = ')(*&^flash@#$%a'
             tm = self._download_json(
-                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
+                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id,
+                note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
+            )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
             return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
         video_urls_dict = {}
-        for i in data['vp']['tkl'][0]['vs']:
-            if 0 < int(i['bid']) <= 10:
-                format_id = self.get_format(i['bid'])
+        for format_item in data['vp']['tkl'][0]['vs']:
+            if 0 < int(format_item['bid']) <= 10:
+                format_id = self.get_format(format_item['bid'])
             else:
                 continue
 
             video_urls = []
 
-            video_urls_info = i['fs']
-            if not i['fs'][0]['l'].startswith('/'):
-                t = get_encode_code(i['fs'][0]['l'])
+            video_urls_info = format_item['fs']
+            if not format_item['fs'][0]['l'].startswith('/'):
+                t = get_encode_code(format_item['fs'][0]['l'])
                 if t.endswith('mp4'):
-                    video_urls_info = i['flvs']
+                    video_urls_info = format_item['flvs']
 
-            for ii in video_urls_info:
-                vl = ii['l']
+            for segment_index, segment in enumerate(video_urls_info):
+                vl = segment['l']
                 if not vl.startswith('/'):
                     vl = get_encode_code(vl)
                 key = get_path_key(
-                    vl.split('/')[-1].split('.')[0])
-                filesize = ii['b']
+                    vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                filesize = segment['b']
                 base_url = data['vp']['du'].split('/')
                 base_url.insert(-1, key)
                 base_url = '/'.join(base_url)
@@ -91,7 +93,9 @@ class IqiyiIE(InfoExtractor):
                 }
                 api_video_url = base_url + vl + '?' + \
                     compat_urllib_parse.urlencode(param)
-                js = self._download_json(api_video_url, video_id)
+                js = self._download_json(
+                    api_video_url, video_id,
+                    note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
                 video_url = js['l']
                 video_urls.append(
                     (video_url, filesize))

From c4ee87022bd18863fc3f22f80064453e272d956f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:57:05 +0800
Subject: [PATCH 0825/1515] [iqiyi] Change id for multipart videos

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c17e1fde4..840cc9a4d 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -207,7 +207,7 @@ class IqiyiIE(InfoExtractor):
             self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
-                    'id': '_part%d' % (i + 1),
+                    'id': '%s_part%d' % (video_id, i + 1),
                     'title': title,
                 }
             )

From 99481135907b5fa3558d4f176fd02acbdafccdb6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:09:33 +0800
Subject: [PATCH 0826/1515] [iqiyi] Add a multipart test case

---
 youtube_dl/extractor/iqiyi.py | 67 +++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 840cc9a4d..d73687d88 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -19,7 +19,7 @@ class IqiyiIE(InfoExtractor):
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
         'md5': '2cb594dc2781e6c941a110d8f358118b',
         'info_dict': {
@@ -27,7 +27,70 @@ class IqiyiIE(InfoExtractor):
             'title': '美国德州空中惊现奇异云团 酷似UFO',
             'ext': 'f4v',
         }
-    }
+    }, {
+        'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
+        'info_dict': {
+            'id': 'e3f585b550a280af23c98b6cb2be19fb',
+            'title': '名侦探柯南第752集',
+        },
+        'playlist': [{
+            'md5': '7e49376fecaffa115d951634917fe105',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '41b75ba13bb7ac0e411131f92bc4f6ca',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '0cee1dd0a3d46a83e71e2badeae2aab0',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '4f8ad72373b0c491b582e7c196b0b1f9',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': 'd89ad028bcfad282918e8098e811711d',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '9cb1e5c95da25dff0660c32ae50903b7',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '155116e0ff1867bbc9b98df294faabc9',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '53f5db77622ae14fa493ed2a278a082b',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }],
+    }]
 
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):

From 865ab62f43eb94a9f4f757a464df147e983cb439 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:13:22 +0800
Subject: [PATCH 0827/1515] [iqiyi] Make _VALID_URL more accurate

v_* urls are individual videos, while a_* urls are playlists, which are
not supported yet.
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d73687d88..f0d423331 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -17,7 +17,7 @@ from ..utils import ExtractorError
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From 08bb8ef2011d795948d8e89478bf3afe4b99405f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:25:00 +0800
Subject: [PATCH 0828/1515] [iqiyi] Unify get_format() and get_bid()

---
 youtube_dl/extractor/iqiyi.py | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f0d423331..122f33692 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -92,6 +92,15 @@ class IqiyiIE(InfoExtractor):
         }],
     }]
 
+    _FORMATS_MAP = [
+        ('1', 'h6'),
+        ('2', 'h5'),
+        ('3', 'h4'),
+        ('4', 'h3'),
+        ('5', 'h2'),
+        ('10', 'h1'),
+    ]
+
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
@@ -167,27 +176,12 @@ class IqiyiIE(InfoExtractor):
         return video_urls_dict
 
     def get_format(self, bid):
-        _dict = {
-            '1': 'h6',
-            '2': 'h5',
-            '3': 'h4',
-            '4': 'h3',
-            '5': 'h2',
-            '10': 'h1'
-        }
-        return _dict.get(str(bid), None)
+        matched_format_ids = [_format_id for _bid, _format_id in self._FORMATS_MAP if _bid == str(bid)]
+        return matched_format_ids[0] if len(matched_format_ids) else None
 
     def get_bid(self, format_id):
-        _dict = {
-            'h6': '1',
-            'h5': '2',
-            'h4': '3',
-            'h3': '4',
-            'h2': '5',
-            'h1': '10',
-            'best': 'best'
-        }
-        return _dict.get(format_id, None)
+        matched_bids = [_bid for _bid, _format_id in self._FORMATS_MAP if _format_id == format_id]
+        return matched_bids[0] if len(matched_bids) else None
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))

From 9c5f685ef14a8b44d17b897ba8ae2da051011c35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:39:03 +0800
Subject: [PATCH 0829/1515] [iqiyi] Improve regex pattern again

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 122f33692..15481b84b 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -223,7 +223,7 @@ class IqiyiIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
         swf_url = self._search_regex(
-            r'(http://.+?MainPlayer.+?\.swf)', webpage, 'swf player URL')
+            r'(http://[^\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
         enc_key = self.get_enc_key(swf_url, video_id)

From b5a3c7f10927c9d55f6fdad5f5c002e02338642e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:47:36 +0800
Subject: [PATCH 0830/1515] [iqiyi] Cache encryption keys

---
 youtube_dl/extractor/iqiyi.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 15481b84b..9106dd074 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
+import os.path
 import random
 import re
 import time
@@ -11,7 +12,10 @@ import zlib
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    url_basename,
+)
 
 
 class IqiyiIE(InfoExtractor):
@@ -207,12 +211,20 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
+        filename, _ = os.path.splitext(url_basename(swf_url))
+        enc_key_json = self._downloader.cache.load('iqiyi-enc-key', filename)
+        if enc_key_json is not None:
+            return enc_key_json[0]
+
         req = self._request_webpage(
             swf_url, video_id, note='download swf content')
         cn = req.read()
         cn = zlib.decompress(cn[8:])
         pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
         enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
+
+        self._downloader.cache.store('iqiyi-enc-key', filename, [enc_key])
+
         return enc_key
 
     def _real_extract(self, url):

From d00735a0c5aabd38b37bfea76a93ae8c47a8d419 Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sat, 6 Jun 2015 23:01:23 +0300
Subject: [PATCH 0831/1515] [ruutu] Don't use fallback for DASH and other
 non-HTTP urls

---
 youtube_dl/extractor/ruutu.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index e346434f9..59e0b12fd 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -62,6 +62,8 @@ class RuutuIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
                 parsed_urls.append(url)
             else:
+                if not fmt.tag.startswith('HTTP'):
+                    continue
                 proto = compat_urllib_parse_urlparse(url).scheme
                 width_str, height_str = fmt.get('resolution').split('x')
                 tbr = int(fmt.get('bitrate', 0))

From de390ea0771a0e35c0c2970bc00f5fa2dd9d3eac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 7 Jun 2015 00:19:45 +0200
Subject: [PATCH 0832/1515] update: Use https for getting the version info
 (fixes #5909)

---
 youtube_dl/update.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index de3169eef..fc7ac8305 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -50,7 +50,7 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
+    UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

From 9414338a48ca815fd666aad496ebabd6d0c76e5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jun 2015 05:37:29 +0600
Subject: [PATCH 0833/1515] [ruutu] Improve, make more robust and fix python
 2.6 support

---
 youtube_dl/extractor/ruutu.py | 129 +++++++++++++++++++++-------------
 1 file changed, 79 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 59e0b12fd..4e22628d0 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -3,88 +3,117 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
-import re
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    xpath_text,
+)
 
 
 class RuutuIE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?ruutu\.fi/ohjelmat/(?:[^/]+/)?(?P<id>.*)$'
+    _VALID_URL = r'http://(?:www\.)?ruutu\.fi/ohjelmat/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
     _TESTS = [
         {
             'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
             'md5': 'ab2093f39be1ca8581963451b3c0234f',
             'info_dict': {
-                'id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+                'id': '2058907',
+                'display_id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
                 'ext': 'mp4',
                 'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
-                'description': 'Toinen toistaan huikeampia ohjelmaideoita ja täysin päätöntä sekoilua? No sitä juuri nimenomaan. Metro Helsingin Iltapäivän vieraaksi saapui Tuomas Kauhanen ja he Petra Kalliomaan kanssa keskustelivat hieman ennen lähetyksen alkua, mutta kamerat olivatkin jo päällä.',
+                'description': 'md5:cfc6ccf0e57a814360df464a91ff67d6',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 114,
+                'age_limit': 0,
             },
-            'params': {
-                'format': 'http-1000',
-            }
         },
         {
             'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
-                'id': 'superpesis-katso-koko-kausi-ruudussa',
+                'id': '2057306',
+                'display_id': 'superpesis-katso-koko-kausi-ruudussa',
                 'ext': 'mp4',
                 'title': 'Superpesis: katso koko kausi Ruudussa',
-                'description': 'Huippujännittävän Superpesiksen suoria ottelulähetyksiä seurataan Ruudussa kauden alusta viimeiseen finaaliin asti. Katso lisätiedot osoitteesta ruutu.fi/superpesis.',
+                'description': 'md5:44c44a99fdbe5b380ab74ebd75f0af77',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 40,
+                'age_limit': 0,
             },
-            'params': {
-                'format': 'http-1000',
-            }
         },
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        display_id = self._match_id(url)
 
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        media_id = self._html_search_regex(r'data-media-id="(\d+)"', webpage, 'media_id')
-        media_json = self._parse_json(self._search_regex(r'jQuery.extend\([^,]+, (.*)\);', webpage, 'media_data'), video_id)
-        xml_url = media_json['ruutuplayer']['xmlUrl'].replace('{ID}', media_id)
-        media_xml = self._download_xml(xml_url, media_id)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'data-media-id="(\d+)"', webpage, 'media id')
+
+        video_xml_url = None
+
+        media_data = self._search_regex(
+            r'jQuery\.extend\([^,]+,\s*(.+?)\);', webpage,
+            'media data', default=None)
+        if media_data:
+            media_json = self._parse_json(media_data, display_id, fatal=False)
+            if media_json:
+                xml_url = media_json.get('ruutuplayer', {}).get('xmlUrl')
+                if xml_url:
+                    video_xml_url = xml_url.replace('{ID}', video_id)
+
+        if not video_xml_url:
+            video_xml_url = 'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id
+
+        video_xml = self._download_xml(video_xml_url, video_id)
 
         formats = []
-        parsed_urls = []
-        for fmt in media_xml.findall('.//Clip//'):
-            url = fmt.text
-            if not fmt.tag.endswith('File') or url in parsed_urls or \
-                    'NOT_USED' in url:
-                continue
+        processed_urls = []
 
-            if url.endswith('m3u8'):
-                formats.extend(self._extract_m3u8_formats(url, media_id, m3u8_id='hls'))
-                parsed_urls.append(url)
-            elif url.endswith('f4m'):
-                formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
-                parsed_urls.append(url)
-            else:
-                if not fmt.tag.startswith('HTTP'):
-                    continue
-                proto = compat_urllib_parse_urlparse(url).scheme
-                width_str, height_str = fmt.get('resolution').split('x')
-                tbr = int(fmt.get('bitrate', 0))
-                formats.append({
-                    'format_id': '%s-%d' % (proto, tbr),
-                    'url': url,
-                    'width': int(width_str),
-                    'height': int(height_str),
-                    'tbr': tbr,
-                    'ext': url.rsplit('.', 1)[-1],
-                    'live': True,
-                    'protocol': proto,
-                })
+        def extract_formats(node):
+            for child in node:
+                if child.tag.endswith('Files'):
+                    extract_formats(child)
+                elif child.tag.endswith('File'):
+                    video_url = child.text
+                    if not video_url or video_url in processed_urls or 'NOT_USED' in video_url:
+                        return
+                    processed_urls.append(video_url)
+                    ext = determine_ext(video_url)
+                    if ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls'))
+                    elif ext == 'f4m':
+                        formats.extend(self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds'))
+                    else:
+                        proto = compat_urllib_parse_urlparse(video_url).scheme
+                        if not child.tag.startswith('HTTP') and proto != 'rtmp':
+                            continue
+                        preference = -1 if proto == 'rtmp' else 1
+                        label = child.get('label')
+                        tbr = int_or_none(child.get('bitrate'))
+                        width, height = [int_or_none(x) for x in child.get('resolution', '').split('x')]
+                        formats.append({
+                            'format_id': '%s-%s' % (proto, label if label else tbr),
+                            'url': video_url,
+                            'width': width,
+                            'height': height,
+                            'tbr': tbr,
+                            'preference': preference,
+                        })
+
+        extract_formats(video_xml.find('./Clip'))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': self._og_search_title(webpage),
-            'formats': formats,
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': int(media_xml.find('.//Runtime').text),
-            'age_limit': int(media_xml.find('.//AgeLimit').text),
+            'duration': int_or_none(xpath_text(video_xml, './/Runtime', 'duration')),
+            'age_limit': int_or_none(xpath_text(video_xml, './/AgeLimit', 'age limit')),
+            'formats': formats,
         }

From 9836cfb8d682c91036ce417fa31200673b52115b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jun 2015 08:12:21 +0600
Subject: [PATCH 0834/1515] [options] Clarify `--list-extractors` (Closes
 #5916)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5a2315bd9..689fa7595 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -145,7 +145,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
-        help='List all supported extractors and the URLs they would handle')
+        help='List all supported extractors')
     general.add_option(
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,

From b26733ba7f376f8c9285ac7928534286622bbc7c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 15:29:17 +0800
Subject: [PATCH 0835/1515] [brightcove] Allow single quotes in Brightcove URLs
 (fixes #5901)

---
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/generic.py    | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4f60d5366..c1d4320e1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -172,7 +172,7 @@ class BrightcoveIE(InfoExtractor):
         """Return a list of all Brightcove URLs from the webpage """
 
         url_m = re.search(
-            r'<meta\s+property="og:video"\s+content="(https?://(?:secure|c)\.brightcove.com/[^"]+)"',
+            r'<meta\s+property=[\'"]og:video[\'"]\s+content=[\'"](https?://(?:secure|c)\.brightcove.com/[^\'"]+)[\'"]',
             webpage)
         if url_m:
             url = unescapeHTML(url_m.group(1))
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 96ca398de..759691365 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -789,6 +789,18 @@ class GenericIE(InfoExtractor):
                 # rtmpe downloads
                 'skip_download': True,
             }
+        },
+        # Brightcove URL in single quotes
+        {
+            'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
+            'md5': '4ae374f1f8b91c889c4b9203c8c752af',
+            'info_dict': {
+                'id': '4255764656001',
+                'ext': 'mp4',
+                'title': 'SN Presents: Russell Martin, World Citizen',
+                'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
+                'uploader': 'Rogers Sportsnet',
+            },
         }
     ]
 

From 621ed9f5f4d9d82659272ebe01e740e9196fad61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:33:22 +0800
Subject: [PATCH 0836/1515] [common] Add note and errnote field for
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cecf917ff..49e4dc710 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,7 @@ class InfoExtractor(object):
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
-                              m3u8_id=None):
+                              m3u8_id=None, note=None, errnote=None):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -865,8 +865,8 @@ class InfoExtractor(object):
 
         m3u8_doc = self._download_webpage(
             m3u8_url, video_id,
-            note='Downloading m3u8 information',
-            errnote='Failed to download m3u8 information')
+            note=note or 'Downloading m3u8 information',
+            errnote=errnote or 'Failed to download m3u8 information')
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 65ba8b23f471b96e6f937f2754c729e22bf2cf0a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:34:19 +0800
Subject: [PATCH 0837/1515] [discovery] Rewrite DiscoveryIE (fixes #5898)

Discovery.com now uses a completely different approach for serving
videos. At least in both test cases brightcove are involved. However,
AMF support is necessary for these brightcove videos. As a result, I
try to extract videos from the info page ('?flat=1'). The downloaded
file can be different from the one in browsers.
---
 youtube_dl/extractor/discovery.py | 52 ++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index d3e667528..d6723ecf2 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -2,19 +2,19 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    parse_duration,
     parse_iso8601,
-    int_or_none,
 )
+from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
     _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
-        'md5': '3c69d77d9b0d82bfd5e5932a60f26504',
         'info_dict': {
-            'id': 'mission-impossible-outtakes',
-            'ext': 'flv',
+            'id': '20769',
+            'ext': 'mp4',
             'title': 'Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                             ' each other -- to the point of confusing Jamie\'s dog -- and '
@@ -24,22 +24,36 @@ class DiscoveryIE(InfoExtractor):
             'timestamp': 1303099200,
             'upload_date': '20110418',
         },
-    }
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        }
+    }, {
+        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mythbusters-the-simpsons',
+        'info_dict': {
+            'id': 'mythbusters-the-simpsons',
+            'title': 'MythBusters: The Simpsons',
+        },
+        'playlist_count': 9,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        info = self._download_json(url + '?flat=1', video_id)
 
-        info = self._parse_json(self._search_regex(
-            r'(?s)<script type="application/ld\+json">(.*?)</script>',
-            webpage, 'video info'), video_id)
+        video_title = info.get('playlist_title') or info.get('video_title')
 
-        return {
-            'id': video_id,
-            'title': info['name'],
-            'url': info['contentURL'],
-            'description': info.get('description'),
-            'thumbnail': info.get('thumbnailUrl'),
-            'timestamp': parse_iso8601(info.get('uploadDate')),
-            'duration': int_or_none(info.get('duration')),
-        }
+        entries = [{
+            'id': compat_str(video_info['id']),
+            'formats': self._extract_m3u8_formats(
+                video_info['src'], video_id, ext='mp4',
+                note='Download m3u8 information for video %d' % (idx + 1)),
+            'title': video_info['title'],
+            'description': video_info.get('description'),
+            'duration': parse_duration(video_info.get('video_length')),
+            'webpage_url': video_info.get('href'),
+            'thumbnail': video_info.get('thumbnailURL'),
+            'alt_title': video_info.get('secondary_title'),
+            'timestamp': parse_iso8601(video_info.get('publishedDate')),
+        } for idx, video_info in enumerate(info['playlist'])]
+
+        return self.playlist_result(entries, video_id, video_title)

From 68477c3dab97733eb7a2feb8fcc90f648c29c2b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:38:39 +0800
Subject: [PATCH 0838/1515] [tlc] Fix test failure due to DiscoveryIE changes

---
 youtube_dl/extractor/tlc.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 9f9e388c5..13263614c 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -12,17 +12,22 @@ class TlcIE(DiscoveryIE):
     IE_NAME = 'tlc.com'
     _VALID_URL = r'http://www\.tlc\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
 
-    _TEST = {
+    # DiscoveryIE has _TESTS
+    _TESTS = [{
         'url': 'http://www.tlc.com/tv-shows/cake-boss/videos/too-big-to-fly.htm',
-        'md5': 'c4038f4a9b44d0b5d74caaa64ed2a01a',
         'info_dict': {
-            'id': '853232',
+            'id': '104493',
             'ext': 'mp4',
-            'title': 'Cake Boss: Too Big to Fly',
+            'title': 'Too Big to Fly',
             'description': 'Buddy has taken on a high flying task.',
             'duration': 119,
+            'timestamp': 1393365060,
+            'upload_date': '20140225',
         },
-    }
+        'params': {
+            'skip_download': True,  # requires ffmpef
+        },
+    }]
 
 
 class TlcDeIE(InfoExtractor):

From edb99d4c18475ba27fae4f7d0ec6e3db9b574885 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 01:17:21 +0800
Subject: [PATCH 0839/1515] [instagram] Handling null values (fixes #5919)

I didn't add the test case here because it takes too much time. (7
minutes on my machine)
---
 youtube_dl/extractor/instagram.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b10755788..b92367a9d 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -100,7 +100,9 @@ class InstagramUserIE(InfoExtractor):
                 thumbnails_el = it.get('images', {})
                 thumbnail = thumbnails_el.get('thumbnail', {}).get('url')
 
-                title = it.get('caption', {}).get('text', it['id'])
+                # In some cases caption is null, which corresponds to None
+                # in python. As a result, it.get('caption', {}) gives None
+                title = (it.get('caption') or {}).get('text', it['id'])
 
                 entries.append({
                     'id': it['id'],

From e1ec93304dfcf385380feb95a3777c796cc49420 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 01:46:33 +0800
Subject: [PATCH 0840/1515] [instagram:user] Truncate title to 80 characters
 (#5919)

This is a workaround. Currently YoutubeDL.process_info() truncates
info_dict['title'] to 200 characters, but the implementation can't
handle wide characters.
---
 youtube_dl/extractor/instagram.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b92367a9d..3d78f78c4 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    limit_length,
+)
 
 
 class InstagramIE(InfoExtractor):
@@ -106,7 +109,7 @@ class InstagramUserIE(InfoExtractor):
 
                 entries.append({
                     'id': it['id'],
-                    'title': title,
+                    'title': limit_length(title, 80),
                     'formats': formats,
                     'thumbnail': thumbnail,
                     'webpage_url': it.get('link'),

From 788be3313df7ad020dc0a98bd5ed43a60120fb3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 13:32:04 +0800
Subject: [PATCH 0841/1515] [cnet] Fix theplatform vid extraction (fixes #5924)

---
 youtube_dl/extractor/cnet.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3145b3051..5dd69bff7 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CNETIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
@@ -25,7 +25,20 @@ class CNETIE(InfoExtractor):
         'params': {
             'skip_download': 'requires rtmpdump',
         }
-    }
+    }, {
+        'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
+        'info_dict': {
+            'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
+            'ext': 'flv',
+            'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
+            'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
+            'uploader': 'Ashley Esqueda',
+            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -42,7 +55,7 @@ class CNETIE(InfoExtractor):
             raise ExtractorError('Cannot find video data')
 
         mpx_account = data['config']['players']['default']['mpx_account']
-        vid = vdata['files']['rtmp']
+        vid = vdata['files'].get('rtmp', vdata['files']['hds'])
         tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
 
         video_id = vdata['id']

From 01e21b89eefc32bcc4a92c3a82658cee139b6b2c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 17:39:55 +0800
Subject: [PATCH 0842/1515] [noco] Skip invalid timestamps (closes #5826)

---
 youtube_dl/extractor/noco.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 664dc81d4..5bbd2dcf6 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -166,6 +166,10 @@ class NocoIE(InfoExtractor):
         self._sort_formats(formats)
 
         timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')
+
+        if timestamp is not None and timestamp < 0:
+            timestamp = None
+
         uploader = show.get('partner_name')
         uploader_id = show.get('partner_key')
         duration = float_or_none(show.get('duration_ms'), 1000)

From a55e36f48d1f0dc5454b144c7373361f284b9236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Jun 2015 21:05:17 +0600
Subject: [PATCH 0843/1515] [YoutubeDL] Handle out-of-range timestamps (#5826)

---
 youtube_dl/YoutubeDL.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aa6ec9d9a..b1f792d4e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1016,13 +1016,13 @@ class YoutubeDL(object):
             info_dict['display_id'] = info_dict['id']
 
         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
-            # Working around negative timestamps in Windows
-            # (see http://bugs.python.org/issue1646728)
-            if info_dict['timestamp'] < 0 and os.name == 'nt':
-                info_dict['timestamp'] = 0
-            upload_date = datetime.datetime.utcfromtimestamp(
-                info_dict['timestamp'])
-            info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+            # Working around out-of-range timestamp values (e.g. negative ones on Windows,
+            # see http://bugs.python.org/issue1646728)
+            try:
+                upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
+                info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+            except (ValueError, OverflowError, OSError):
+                pass
 
         if self.params.get('listsubtitles', False):
             if 'automatic_captions' in info_dict:

From 627b96482567c1525dddfcceae2c16ff53c18b6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jun 2015 11:41:17 +0800
Subject: [PATCH 0844/1515] [kickstarted] Extract thumbnails in embedded videos
 (#5929)

---
 youtube_dl/extractor/kickstarter.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 7d4b57056..1d391e69f 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -28,6 +28,14 @@ class KickStarterIE(InfoExtractor):
             'uploader': 'Pebble Technology',
             'title': 'Pebble iOS Notifications',
         }
+    }, {
+        'url': 'https://www.kickstarter.com/projects/1420158244/power-drive-2000/widget/video.html',
+        'info_dict': {
+            'id': '1420158244',
+            'ext': 'mp4',
+            'title': 'Power Drive 2000',
+        },
+        'expected_warnings': ['OpenGraph description'],
     }]
 
     def _real_extract(self, url):
@@ -48,10 +56,15 @@ class KickStarterIE(InfoExtractor):
                 'title': title,
             }
 
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
+        if thumbnail is None:
+            thumbnail = self._html_search_regex(
+                r'<img[^>]+class="[^"]+\s*poster\s*[^"]+"[^>]+src="([^"]+)"',
+                webpage, 'thumbnail image', fatal=False)
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
             'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': thumbnail,
         }

From e1b9322b091122b6f6832c70e3a845a84ee1764e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jun 2015 14:48:18 +0800
Subject: [PATCH 0845/1515] [youtube] Restricter DASH signature pattern

A problematic DASH url is:
https://manifest.googlevideo.com/api/manifest/dash/mm/35/key/yt5/ip/140.112.247.145/ms/pm/mv/s/mt/1433794435/id/o-AD2Od_dsOlAUYPu03ZsVWKSbGEbCJJrMp9vnXGhnyRhd/mn/sn-aigllm7r/sparams/as%2Chfr%2Cid%2Cip%2Cipbits%2Citag%2Cmm%2Cmn%2Cms%2Cmv%2Cnh%2Cpl%2Cplayback_host%2Crequiressl%2Csource%2Cexpire/fexp/9406009%2C9406821%2C9407575%2C9408142%2C9408420%2C9408710%2C9409121%2C9409208%2C9412514%2C9412780%2C9413208%2C9413426%2C9413476%2C9413503%2C9415304%2C9415753/upn/viDQrs8SnmE/as/fmp4_audio_clear%2Cwebm_audio_clear%2Cfmp4_sd_hd_clear%2Cwebm_sd_hd_clear%2Cwebm2_sd_hd_clear/playback_host/r4---sn-aigllm7r.googlevideo.com/ipbits/0/requiressl/yes/pl/20/itag/0/source/youtube/expire/1433824806/nh/EAQ/signature/81ABE6391E351BA495F5B041B00FF1257A353318.1A6E48ABB74E8F4AE73CA2CB1F963FC34E33DEE7/sver/3/hfr/1
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 419f7b019..083da777d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -785,7 +785,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
             return '/signature/%s' % dec_s
-        dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
+        dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
         dash_doc = self._download_xml(
             dash_manifest_url, video_id,
             note='Downloading DASH manifest',

From d9cf48e81e38f4bf151a8648c48d6e5233325b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 20:36:08 +0600
Subject: [PATCH 0846/1515] [spiegeltv] Extract all formats and prefer hls
 (Closes #5843)

---
 youtube_dl/extractor/spiegeltv.py | 45 ++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 359722ad6..08a5c4314 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -2,7 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import (
+    determine_ext,
+    float_or_none,
+)
 
 
 class SpiegeltvIE(InfoExtractor):
@@ -17,7 +21,7 @@ class SpiegeltvIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg$',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         }
     }, {
@@ -53,7 +57,35 @@ class SpiegeltvIE(InfoExtractor):
         server_json = self._download_json(
             'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
             video_id, note='Downloading server information')
-        server = server_json['streamingserver'][0]['endpoint']
+
+        format = '16x9' if is_wide else '4x3'
+
+        formats = []
+        for streamingserver in server_json['streamingserver']:
+            endpoint = streamingserver.get('endpoint')
+            if not endpoint:
+                continue
+            play_path = 'mp4:%s_spiegeltv_0500_%s.m4v' % (uuid, format)
+            if endpoint.startswith('rtmp'):
+                formats.append({
+                    'url': endpoint,
+                    'format_id': 'rtmp',
+                    'app': compat_urllib_parse_urlparse(endpoint).path[1:],
+                    'play_path': play_path,
+                    'player_path': 'http://prod-static.spiegel.tv/frontend-076.swf',
+                    'ext': 'flv',
+                    'rtmp_live': True,
+                })
+            elif determine_ext(endpoint) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    endpoint.replace('[video]', play_path),
+                    video_id, 'm4v',
+                    preference=1,  # Prefer hls since it allows to workaround georestriction
+                    m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': endpoint,
+                })
 
         thumbnails = []
         for image in media_json['images']:
@@ -65,17 +97,12 @@ class SpiegeltvIE(InfoExtractor):
 
         description = media_json['subtitle']
         duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
-        format = '16x9' if is_wide else '4x3'
-
-        url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
 
         return {
             'id': video_id,
             'title': title,
-            'url': url,
-            'ext': 'm4v',
             'description': description,
             'duration': duration,
             'thumbnails': thumbnails,
-            'rtmp_live': True,
+            'formats': formats,
         }

From 9bf99891d08d166ac1d81b652dc487bb940fa685 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 21:23:53 +0600
Subject: [PATCH 0847/1515] [cbs] Add support for colbertlateshow (Closes
 #5888)

---
 youtube_dl/extractor/cbs.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 1ceb9d8d9..89614a3c9 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class CBSIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)/(?P<id>[^/]+)/.*'
+    _VALID_URL = r'https?://(?:(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/(?P<id>[^/]+)/.*'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
@@ -34,12 +34,18 @@ class CBSIE(InfoExtractor):
             'skip_download': True,
         },
         '_skip': 'Blocked outside the US',
+    }, {
+        'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
+        'only_matching': True,
+    }, {
+        'url': 'http://colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         real_id = self._search_regex(
-            r"video\.settings\.pid\s*=\s*'([^']+)';",
+            [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
         return self.url_result('theplatform:%s' % real_id)

From 9d581f3d5224140ca35ebd06d614b929e9252cd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 21:39:45 +0600
Subject: [PATCH 0848/1515] [cbs] Extract display_id

---
 youtube_dl/extractor/cbs.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 89614a3c9..75fffb156 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,12 +4,13 @@ from .common import InfoExtractor
 
 
 class CBSIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/(?P<id>[^/]+)/.*'
+    _VALID_URL = r'https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
         'info_dict': {
             'id': '4JUVEwq3wUT7',
+            'display_id': 'connect-chat-feat-garth-brooks',
             'ext': 'flv',
             'title': 'Connect Chat feat. Garth Brooks',
             'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
@@ -24,6 +25,7 @@ class CBSIE(InfoExtractor):
         'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/',
         'info_dict': {
             'id': 'WWF_5KqY3PK1',
+            'display_id': 'st-vincent',
             'ext': 'flv',
             'title': 'Live on Letterman - St. Vincent',
             'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.',
@@ -38,14 +40,19 @@ class CBSIE(InfoExtractor):
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
         'only_matching': True,
     }, {
-        'url': 'http://colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
+        'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
         real_id = self._search_regex(
             [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
-        return self.url_result('theplatform:%s' % real_id)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': 'theplatform:%s' % real_id,
+            'display_id': display_id,
+        }

From 6e054aacca2ac44413ed37ee8b1d63a09c8b4ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 23:07:22 +0600
Subject: [PATCH 0849/1515] [theplatform] Take care of /select/media URLs
 (Closes #5746)

---
 youtube_dl/extractor/theplatform.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 92731ad3d..48c6ff03f 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -26,7 +26,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
+           (?:(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)|(?P<media>(?:[^/]+/)+select/media/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{
@@ -56,6 +56,17 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
+        'info_dict': {
+            'id': 'yMBg9E8KFxZD',
+            'ext': 'mp4',
+            'description': 'md5:644ad9188d655b742f942bf2e06b002d',
+            'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
+        }
+    }, {
+        'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -85,6 +96,11 @@ class ThePlatformIE(InfoExtractor):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
+        path = provider_id
+        if mobj.group('media'):
+            path += '/media'
+        path += '/' + video_id
+
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
         elif mobj.group('config'):
@@ -94,8 +110,7 @@ class ThePlatformIE(InfoExtractor):
             config = self._download_json(config_url, video_id, 'Downloading config')
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = ('http://link.theplatform.com/s/{0}/{1}/meta.smil?'
-                        'format=smil&mbr=true'.format(provider_id, video_id))
+            smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
 
         sig = smuggled_data.get('sig')
         if sig:
@@ -112,7 +127,7 @@ class ThePlatformIE(InfoExtractor):
         else:
             raise ExtractorError(error_msg, expected=True)
 
-        info_url = 'http://link.theplatform.com/s/{0}/{1}?format=preview'.format(provider_id, video_id)
+        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 

From bd5bc0cd5af257abf7a1a4c14a9dd39c4f97e622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 23:12:13 +0600
Subject: [PATCH 0850/1515] [theplatform] Check for /select/media URLs first
 (#5746)

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 48c6ff03f..83d833e30 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -26,7 +26,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)|(?P<media>(?:[^/]+/)+select/media/))?
+           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 70219b0f4371fe54cc72d025ce06fc4691ba12fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 9 Jun 2015 23:49:11 +0200
Subject: [PATCH 0851/1515] [youtube:playlist] Use an iterator for the entries
 (closes #5935)

So that '--playlist-end' downloads only the required pages.
---
 youtube_dl/extractor/youtube.py | 47 ++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 083da777d..3448bec4f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1290,7 +1290,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     def _extract_playlist(self, playlist_id):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
-        more_widget_html = content_html = page
 
         for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
             match = match.strip()
@@ -1310,36 +1309,36 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 self.report_warning('Youtube gives an alert message: ' + match)
 
         # Extract the video ids from the playlist pages
-        ids = []
+        def _entries():
+            more_widget_html = content_html = page
+            for page_num in itertools.count(1):
+                matches = re.finditer(self._VIDEO_RE, content_html)
+                # We remove the duplicates and the link with index 0
+                # (it's not the first video of the playlist)
+                new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
+                for vid_id in new_ids:
+                    yield self.url_result(vid_id, 'Youtube', video_id=vid_id)
 
-        for page_num in itertools.count(1):
-            matches = re.finditer(self._VIDEO_RE, content_html)
-            # We remove the duplicates and the link with index 0
-            # (it's not the first video of the playlist)
-            new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
-            ids.extend(new_ids)
+                mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+                if not mobj:
+                    break
 
-            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-            if not mobj:
-                break
-
-            more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
-                'Downloading page #%s' % page_num,
-                transform_source=uppercase_escape)
-            content_html = more['content_html']
-            if not content_html.strip():
-                # Some webpages show a "Load more" button but they don't
-                # have more videos
-                break
-            more_widget_html = more['load_more_widget_html']
+                more = self._download_json(
+                    'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+                    'Downloading page #%s' % page_num,
+                    transform_source=uppercase_escape)
+                content_html = more['content_html']
+                if not content_html.strip():
+                    # Some webpages show a "Load more" button but they don't
+                    # have more videos
+                    break
+                more_widget_html = more['load_more_widget_html']
 
         playlist_title = self._html_search_regex(
             r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
             page, 'title')
 
-        url_results = self._ids_to_results(ids)
-        return self.playlist_result(url_results, playlist_id, playlist_title)
+        return self.playlist_result(_entries(), playlist_id, playlist_title)
 
     def _real_extract(self, url):
         # Extract playlist id

From d84f1d14b526c4a5359117a58f25691a3da4c97e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 9 Jun 2015 22:08:16 -0300
Subject: [PATCH 0852/1515] Adds support for XviD output with extra
 parametrization

As the "LG Time Machine" (a (not so) smart TV) has a limitation for video dimensions (as for codecs), I take to implement an extra parameter `--pp-params` where we can send extra parameterization for the video converter (post-processor).

Example:
```
$ youtube-dl --recode-video=xvid --pp-params='-s 720x480' -c https://www.youtube.com/watch?v=BE7Qoe2ZiXE
```
That works fine on a 4yo LG Time Machine.

Closes #5733
---
 README.md                          |  3 ++-
 youtube_dl/__init__.py             |  5 ++++-
 youtube_dl/options.py              |  6 +++++-
 youtube_dl/postprocessor/ffmpeg.py | 14 ++++++++++----
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index f3d83c89f..726ec9cf2 100644
--- a/README.md
+++ b/README.md
@@ -213,7 +213,8 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
+    --pp-params                      Extra parameters for video post-processor. The params will be splited on spaces.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ace17857c..5b28e4817 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -169,8 +169,10 @@ def _real_main(argv=None):
         if not opts.audioquality.isdigit():
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
-        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
+    if opts.pp_params is not None:
+        opts.pp_params = opts.pp_params.split()
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
@@ -227,6 +229,7 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
+            'extra_params': opts.pp_params
         })
     if opts.convertsubtitles:
         postprocessors.append({
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..ceb4b5f38 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -686,7 +686,11 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
-        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
+    postproc.add_option(
+        '--pp-params',
+        dest='pp_params', default=None,
+        help='Extra parameters for video post-processor. The params will be splited on spaces.')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index cc65b34e7..a696b12b4 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -287,22 +287,28 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None):
+    def __init__(self, downloader=None, preferedformat=None, extra_params=[]):
         super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
+        self._extra_params = extra_params
 
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
-        outpath = prefix + sep + self._preferedformat
+        ext = self._preferedformat
+        options = self._extra_params
+        if self._preferedformat == 'xvid':
+            ext = 'avi'
+            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
+        outpath = prefix + sep + ext
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
-        self.run_ffmpeg(path, outpath, [])
+        self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath
         information['format'] = self._preferedformat
-        information['ext'] = self._preferedformat
+        information['ext'] = ext
         return [path], information
 
 

From 0c8662d2b6f033ad42f1cc97989d4975629b524b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 13:40:41 +0800
Subject: [PATCH 0853/1515] [youtube] Fix a TypeError caused by
 4da31bd56629054497634d041035e4bd6fcfacbb

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2424ac2c0..a1906eef6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -816,7 +816,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
-                    if len(segment_list):
+                    if segment_list is not None:
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],

From 93dfcb9357b400b4d7e353d0a9db0e0194135b19 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 13:44:54 +0800
Subject: [PATCH 0854/1515] [downloader/dash] Do not pollute ```self```

---
 youtube_dl/downloader/dash.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 5f14658ba..cd84e0b07 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -16,14 +16,14 @@ class DashSegmentsFD(FileDownloader):
         base_url = info_dict['url']
         segment_urls = info_dict['segment_urls']
 
-        self.byte_counter = 0
+        byte_counter = 0
 
         def append_url_to_file(outf, target_url, target_name):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
             req = compat_urllib_request.Request(target_url)
             data = self.ydl.urlopen(req).read()
             outf.write(data)
-            self.byte_counter += len(data)
+            return len(data)
 
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
@@ -35,15 +35,16 @@ class DashSegmentsFD(FileDownloader):
                 outf, combine_url(base_url, info_dict['initialization_url']),
                 'initialization segment')
             for i, segment_url in enumerate(segment_urls):
-                append_url_to_file(
+                segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),
                     'segment %d / %d' % (i + 1, len(segment_urls)))
+                byte_counter += segment_len
 
         self.try_rename(tmpfilename, filename)
 
         self._hook_progress({
-            'downloaded_bytes': self.byte_counter,
-            'total_bytes': self.byte_counter,
+            'downloaded_bytes': byte_counter,
+            'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
         })

From 7ebd5376feb493edd0bc04abd07bba89397b7307 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:15:20 +0800
Subject: [PATCH 0855/1515] [nfl] Relax _VALID_URL (fixes #5940)

---
 youtube_dl/extractor/nfl.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 2684dd250..dc54634a5 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -19,7 +19,7 @@ class NFLIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
         (?:.+?/)*
-        (?P<id>(?:\d[a-z]{2}\d{13}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
+        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
     _TESTS = [
         {
             'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
@@ -58,6 +58,10 @@ class NFLIE(InfoExtractor):
                 'upload_date': '20150202',
             },
         },
+        {
+            'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
+            'only_matching': True,
+        }
     ]
 
     @staticmethod

From 5bf3276e8d6ee7d017c8be04414398752cd9cdf3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:45:54 +0800
Subject: [PATCH 0856/1515] [downloader/dash] Add testing facility

---
 youtube_dl/downloader/dash.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index cd84e0b07..a4685d307 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -16,12 +16,21 @@ class DashSegmentsFD(FileDownloader):
         base_url = info_dict['url']
         segment_urls = info_dict['segment_urls']
 
+        is_test = self.params.get('test', False)
+        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
         byte_counter = 0
 
-        def append_url_to_file(outf, target_url, target_name):
+        def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
             req = compat_urllib_request.Request(target_url)
+            if remaining_bytes is not None:
+                req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
+
             data = self.ydl.urlopen(req).read()
+
+            if remaining_bytes is not None:
+                data = data[:remaining_bytes]
+
             outf.write(data)
             return len(data)
 
@@ -37,8 +46,13 @@ class DashSegmentsFD(FileDownloader):
             for i, segment_url in enumerate(segment_urls):
                 segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),
-                    'segment %d / %d' % (i + 1, len(segment_urls)))
+                    'segment %d / %d' % (i + 1, len(segment_urls)),
+                    remaining_bytes)
                 byte_counter += segment_len
+                if remaining_bytes is not None:
+                    remaining_bytes -= segment_len
+                    if remaining_bytes <= 0:
+                        break
 
         self.try_rename(tmpfilename, filename)
 

From 8a1a26ce4c64d7a2c142718fc56f46d9a1c2c4f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:47:02 +0800
Subject: [PATCH 0857/1515] [youtube] Add a test for the DASH segment
 downloader

---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a1906eef6..939f5e61f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -516,6 +516,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # DASH manifest with segment_list
+        {
+            'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
+            'md5': '8ce563a1d667b599d21064e982ab9e31',
+            'info_dict': {
+                'id': 'CsmdDsKjzN8',
+                'ext': 'mp4',
+                'upload_date': '20150510',
+                'uploader': 'Airtek',
+                'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
+                'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
+                'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
+            },
+            'params': {
+                'youtube_include_dash_manifest': True,
+                'format': '135',  # bestvideo
+            }
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From eb8be1fe76a9fbc285e6c957b3fdd5c05135ae3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 10 Jun 2015 14:12:43 +0200
Subject: [PATCH 0858/1515] [rtbf] Extract all formats (closes #5947)

---
 youtube_dl/extractor/rtbf.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index 5a381d9ce..e4215d546 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -21,6 +21,13 @@ class RTBFIE(InfoExtractor):
         }
     }
 
+    _QUALITIES = [
+        ('mobile', 'mobile'),
+        ('web', 'SD'),
+        ('url', 'MD'),
+        ('high', 'HD'),
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -32,14 +39,21 @@ class RTBFIE(InfoExtractor):
                 r'data-video="([^"]+)"', webpage, 'data video')),
             video_id)
 
-        video_url = data.get('downloadUrl') or data.get('url')
-
         if data.get('provider').lower() == 'youtube':
+            video_url = data.get('downloadUrl') or data.get('url')
             return self.url_result(video_url, 'Youtube')
+        formats = []
+        for key, format_id in self._QUALITIES:
+            format_url = data['sources'].get(key)
+            if format_url:
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                })
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
             'thumbnail': data.get('thumbnail'),

From f98470df690d053e45691ede2751ab6a4063082b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Jun 2015 23:01:12 +0600
Subject: [PATCH 0859/1515] [bilibili] Fix FutureWarning

---
 youtube_dl/extractor/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 2103ed73a..bf60450c2 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -105,7 +105,7 @@ class BiliBiliIE(InfoExtractor):
                 'filesize': int_or_none(
                     lq_durl.find('./size'), get_attr='text'),
             }]
-            if hq_durl:
+            if hq_durl is not None:
                 formats.append({
                     'format_id': 'hq',
                     'quality': 2,

From a9d56c684319eaf8b9494bd8d2dc9d0f40485254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 19:03:22 +0600
Subject: [PATCH 0860/1515] [rtlnl] Improve _VALID_URL (#5950)

---
 youtube_dl/extractor/rtlnl.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index cfce4550a..41d202c28 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -12,10 +12,10 @@ class RtlNlIE(InfoExtractor):
     IE_NAME = 'rtl.nl'
     IE_DESC = 'rtl.nl and rtlxl.nl'
     _VALID_URL = r'''(?x)
-        https?://(www\.)?
+        https?://(?:www\.)?
         (?:
             rtlxl\.nl/\#!/[^/]+/|
-            rtl\.nl/system/videoplayer/[^?#]+?/video_embed\.html\#uuid=
+            rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -43,6 +43,9 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 97b570a94cc2387153af525f781e144bb4bb791e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 19:04:12 +0600
Subject: [PATCH 0861/1515] [generic] Improve rtl.nl embeds detection (Closes
 #5950)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 759691365..75526384f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,7 +1073,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded rtl.nl player
         matches = re.findall(
-            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
+            r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
             webpage)
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')

From ff0f0b9172e432ebbfca88da91278554eb47c307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 22:18:08 +0600
Subject: [PATCH 0862/1515] [tube8] Fix extraction (Closes #5952)

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 6ca8840b0..c9cb69333 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -47,7 +47,7 @@ class Tube8IE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
-            r'flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
+            r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'))
 
         video_url = flashvars['video_url']
         if flashvars.get('encrypted') is True:

From 99ac0390f559aa6dd09ffd8a15b9b562fda5f363 Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Mon, 8 Jun 2015 05:58:41 +0300
Subject: [PATCH 0863/1515] [fivetv] Add extractor (Closes #5794)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fivetv.py   | 67 ++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/fivetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 67eb96057..d10275d03 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -152,6 +152,7 @@ from .fc2 import FC2IE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
+from .fivetv import FiveTVIE
 from .fktv import (
     FKTVIE,
     FKTVPosteckeIE,
diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
new file mode 100644
index 000000000..e47383b39
--- /dev/null
+++ b/youtube_dl/extractor/fivetv.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class FiveTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?5-tv\.ru/[^/]*/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://5-tv.ru/news/96814/',
+            'md5': 'bbff554ad415ecf5416a2f48c22d9283',
+            'info_dict': {
+                'id': '96814',
+                'ext': 'mp4',
+                'title': 'Россияне выбрали имя для общенациональной платежной системы',
+                'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'width': 480,
+                'height': 360,
+                'duration': 180,
+            },
+        },
+        {
+            'url': 'http://5-tv.ru/video/1021729/',
+            'md5': '299c8b72960efc9990acd2c784dc2296',
+            'info_dict': {
+                'id': '1021729',
+                'ext': 'mp4',
+                'title': '3D принтер',
+                'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'width': 480,
+                'height': 360,
+                'duration': 180,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_link = self._search_regex(
+            r'(<a.*?class="videoplayer">)', webpage, 'video link')
+
+        url = self._search_regex(r'href="([^"]+)"', video_link, 'video url')
+        width = int_or_none(self._search_regex(
+            r'width:(\d+)px', video_link, 'width', default=None, fatal=False))
+        height = int_or_none(self._search_regex(
+            r'height:(\d+)px', video_link, 'height', default=None, fatal=False))
+        duration = int_or_none(self._og_search_property(
+            'video:duration', webpage, 'duration'))
+        return {
+            'id': video_id,
+            'url': url,
+            'width': width,
+            'height': height,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+        }

From 87446dc6186c7e4247fd7f9bc1046ef41f5d1a0f Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sun, 7 Jun 2015 17:25:30 +0300
Subject: [PATCH 0864/1515] [tvc] Add extractor (Closes #5795)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tvc.py      | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tvc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 67eb96057..8c4e12904 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -582,6 +582,7 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv4 import TV4IE
+from .tvc import TVCIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
new file mode 100644
index 000000000..b62ab857c
--- /dev/null
+++ b/youtube_dl/extractor/tvc.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    str_or_none,
+)
+
+
+class TVCIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
+            'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba',
+            'info_dict': {
+                'id': '74622',
+                'display_id': '39702',
+                'ext': 'mp4',
+                'title': 'События. "События". Эфир от 22.05.2015 14:30',
+                'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 1122,
+            },
+        },
+        {
+            'url': 'http://www.tvc.ru/news/show/id/69944',
+            'md5': 'b173128ee7b88b5b06c84e5f7880909f',
+            'info_dict': {
+                'id': '75399',
+                'display_id': '69944',
+                'ext': 'mp4',
+                'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
+                'description': 'md5:f675c8eaf23aab9df542d31773ed6518',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 278,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._og_search_video_url(webpage)
+
+        video_id = self._search_regex(
+            r'video/iframe/id/(\d+)/', video_url, 'video id')
+
+        video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id)
+
+        video_json = self._download_json(video_json_url, video_id)
+
+        formats = []
+        for info in video_json.get('path', {}).get('quality', []):
+            format_id = self._search_regex(
+                r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id',
+                fatal=False)
+            formats.append({
+                'format_id': str_or_none(format_id),
+                'url': info.get('url'),
+                'width': int_or_none(info.get('width')),
+                'height': int_or_none(info.get('height')),
+                'tbr': int_or_none(info.get('bitrate')),
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int_or_none(video_json.get('duration')),
+            'formats': formats,
+        }

From 9f15bdabc85add582d78a6dd57cfbb56cb33baff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:13:36 +0600
Subject: [PATCH 0865/1515] [tvc] Separate embed extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/tvc.py      | 125 ++++++++++++++++++-------------
 2 files changed, 77 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6dc3cbff4..a8d3a8928 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -584,7 +584,10 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv4 import TV4IE
-from .tvc import TVCIE
+from .tvc import (
+    TVCIE,
+    TVCEmbedIE,
+)
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index b62ab857c..0055f9598 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -3,77 +3,98 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     int_or_none,
-    str_or_none,
 )
 
 
-class TVCIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
-            'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba',
-            'info_dict': {
-                'id': '74622',
-                'display_id': '39702',
-                'ext': 'mp4',
-                'title': 'События. "События". Эфир от 22.05.2015 14:30',
-                'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'duration': 1122,
-            },
+class TVCEmbedIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
+        'md5': 'bbc5ff531d1e90e856f60fc4b3afd708',
+        'info_dict': {
+            'id': '74622',
+            'ext': 'mp4',
+            'title': 'События. "События". Эфир от 22.05.2015 14:30',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1122,
         },
-        {
-            'url': 'http://www.tvc.ru/news/show/id/69944',
-            'md5': 'b173128ee7b88b5b06c84e5f7880909f',
-            'info_dict': {
-                'id': '75399',
-                'display_id': '69944',
-                'ext': 'mp4',
-                'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
-                'description': 'md5:f675c8eaf23aab9df542d31773ed6518',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'duration': 278,
-            },
-        },
-    ]
+    }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_url = self._og_search_video_url(webpage)
-
-        video_id = self._search_regex(
-            r'video/iframe/id/(\d+)/', video_url, 'video id')
-
-        video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id)
-
-        video_json = self._download_json(video_json_url, video_id)
+        video = self._download_json(
+            'http://www.tvc.ru/video/json/id/%s' % video_id, video_id)
 
         formats = []
-        for info in video_json.get('path', {}).get('quality', []):
+        for info in video.get('path', {}).get('quality', []):
+            video_url = info.get('url')
+            if not video_url:
+                continue
             format_id = self._search_regex(
-                r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id',
-                fatal=False)
+                r'cdnvideo/([^/]+?)(?:-[^/]+?)?/', video_url,
+                'format id', default=None)
             formats.append({
-                'format_id': str_or_none(format_id),
-                'url': info.get('url'),
+                'url': video_url,
+                'format_id': format_id,
                 'width': int_or_none(info.get('width')),
                 'height': int_or_none(info.get('height')),
                 'tbr': int_or_none(info.get('bitrate')),
             })
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'display_id': display_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': int_or_none(video_json.get('duration')),
+            'title': video['title'],
+            'thumbnail': video.get('picture'),
+            'duration': int_or_none(video.get('duration')),
             'formats': formats,
         }
+
+
+class TVCIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
+    _TESTS = [{
+        'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
+        'info_dict': {
+            'id': '74622',
+            'ext': 'mp4',
+            'title': 'События. "События". Эфир от 22.05.2015 14:30',
+            'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1122,
+        },
+    }, {
+        'url': 'http://www.tvc.ru/news/show/id/69944',
+        'info_dict': {
+            'id': '75399',
+            'ext': 'mp4',
+            'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
+            'description': 'md5:f2098f71e21f309e89f69b525fd9846e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 278,
+        },
+    }, {
+        'url': 'http://www.tvc.ru/channel/brand/id/47/show/episodes#',
+        'info_dict': {
+            'id': '2185',
+            'ext': 'mp4',
+            'title': 'Ещё не поздно. Эфир от 03.08.2013',
+            'description': 'md5:51fae9f3f8cfe67abce014e428e5b027',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 3316,
+        },
+    }]
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, self._match_id(url))
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'TVCEmbed',
+            'url': self._og_search_video_url(webpage),
+            'title': clean_html(self._og_search_title(webpage)),
+            'description': clean_html(self._og_search_description(webpage)),
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }

From 29902c8ec016a7128557d47a7413e82d4e022f01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:22:23 +0600
Subject: [PATCH 0866/1515] [tvc:embed] Add embed extraction routine

---
 youtube_dl/extractor/tvc.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 0055f9598..756fec732 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
@@ -22,6 +24,13 @@ class TVCEmbedIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 494f20cbdca8e76e3cb452bb0feabcb855d9b4a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:22:46 +0600
Subject: [PATCH 0867/1515] [extractor/generic] Add support for tvc embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 75526384f..c797c4b52 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -34,6 +34,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
+from .tvc import TVCEmbedIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
@@ -1301,6 +1302,11 @@ class GenericIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
+        # Look for embedded TVC player
+        rutv_url = TVCEmbedIE._extract_url(webpage)
+        if rutv_url:
+            return self.url_result(rutv_url, 'TVCEmbed')
+
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
         if sportbox_urls:

From 954c1d05299ae7c6a51db46c1ac33ddf150266c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:24:13 +0600
Subject: [PATCH 0868/1515] [tvc] Refactor extractor names

---
 youtube_dl/extractor/__init__.py | 2 +-
 youtube_dl/extractor/generic.py  | 4 ++--
 youtube_dl/extractor/tvc.py      | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a8d3a8928..18b1c5e54 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -586,7 +586,7 @@ from .tv2 import (
 from .tv4 import TV4IE
 from .tvc import (
     TVCIE,
-    TVCEmbedIE,
+    TVCArticleIE,
 )
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c797c4b52..507e4a571 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -34,7 +34,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
-from .tvc import TVCEmbedIE
+from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
@@ -1303,7 +1303,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(rutv_url, 'RUTV')
 
         # Look for embedded TVC player
-        rutv_url = TVCEmbedIE._extract_url(webpage)
+        rutv_url = TVCIE._extract_url(webpage)
         if rutv_url:
             return self.url_result(rutv_url, 'TVCEmbed')
 
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 756fec732..36c2a3196 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class TVCEmbedIE(InfoExtractor):
+class TVCIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
@@ -63,7 +63,7 @@ class TVCEmbedIE(InfoExtractor):
         }
 
 
-class TVCIE(InfoExtractor):
+class TVCArticleIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
     _TESTS = [{
         'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',

From 5ccddb7ecfb1015038f2616dd7e0da78a4365c89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:25:26 +0600
Subject: [PATCH 0869/1515] [tvc] Fix ie_key

---
 youtube_dl/extractor/tvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 36c2a3196..6b5d80aee 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -101,7 +101,7 @@ class TVCArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         return {
             '_type': 'url_transparent',
-            'ie_key': 'TVCEmbed',
+            'ie_key': 'TVC',
             'url': self._og_search_video_url(webpage),
             'title': clean_html(self._og_search_title(webpage)),
             'description': clean_html(self._og_search_description(webpage)),

From 2da09ff8b0de3c27a16d9096f5d28d03f44fcf70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:26:31 +0600
Subject: [PATCH 0870/1515] [extractor/generic] Fix tvc ie_key

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 507e4a571..66aceefb8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor):
         # Look for embedded TVC player
         rutv_url = TVCIE._extract_url(webpage)
         if rutv_url:
-            return self.url_result(rutv_url, 'TVCEmbed')
+            return self.url_result(rutv_url, 'TVC')
 
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)

From f37bdbe537134f1ece0819e2aa677b1fec0c1cc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:28:45 +0600
Subject: [PATCH 0871/1515] [extractor/generic] Add test for tvc embed

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 66aceefb8..6be9e6329 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -292,6 +292,15 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # TVC embed
+        {
+            'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
+            'info_dict': {
+                'id': '55304',
+                'ext': 'mp4',
+                'title': 'Дошкольное воспитание',
+            },
+        },
         # SportBox embed
         {
             'url': 'http://www.vestifinance.ru/articles/25753',

From 499a077761b1577857952dc3b541c9f61a8bcade Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 17:48:42 +0600
Subject: [PATCH 0872/1515] [5tv] Improve

---
 youtube_dl/extractor/fivetv.py | 115 +++++++++++++++++++--------------
 1 file changed, 68 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
index e47383b39..13fbc4da2 100644
--- a/youtube_dl/extractor/fivetv.py
+++ b/youtube_dl/extractor/fivetv.py
@@ -1,67 +1,88 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
+from ..utils import int_or_none
 
 
 class FiveTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?5-tv\.ru/[^/]*/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://5-tv.ru/news/96814/',
-            'md5': 'bbff554ad415ecf5416a2f48c22d9283',
-            'info_dict': {
-                'id': '96814',
-                'ext': 'mp4',
-                'title': 'Россияне выбрали имя для общенациональной платежной системы',
-                'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 360,
-                'duration': 180,
-            },
+    _VALID_URL = r'''(?x)
+                    http://
+                        (?:www\.)?5-tv\.ru/
+                        (?:
+                            (?:[^/]+/)+(?P<id>\d+)|
+                            (?P<path>[^/?#]+)(?:[/?#])?
+                        )
+                    '''
+
+    _TESTS = [{
+        'url': 'http://5-tv.ru/news/96814/',
+        'md5': 'bbff554ad415ecf5416a2f48c22d9283',
+        'info_dict': {
+            'id': '96814',
+            'ext': 'mp4',
+            'title': 'Россияне выбрали имя для общенациональной платежной системы',
+            'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 180,
         },
-        {
-            'url': 'http://5-tv.ru/video/1021729/',
-            'md5': '299c8b72960efc9990acd2c784dc2296',
-            'info_dict': {
-                'id': '1021729',
-                'ext': 'mp4',
-                'title': '3D принтер',
-                'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 360,
-                'duration': 180,
-            },
+    }, {
+        'url': 'http://5-tv.ru/video/1021729/',
+        'info_dict': {
+            'id': '1021729',
+            'ext': 'mp4',
+            'title': '3D принтер',
+            'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 180,
         },
-    ]
+    }, {
+        'url': 'http://www.5-tv.ru/glavnoe/#itemDetails',
+        'info_dict': {
+            'id': 'glavnoe',
+            'ext': 'mp4',
+            'title': 'Итоги недели с 8 по 14 июня 2015 года',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }, {
+        'url': 'http://www.5-tv.ru/glavnoe/broadcasts/508645/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/films/1507502/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/programs/broadcast/508713/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/angel/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.5-tv.ru/schedule/?iframe=true&width=900&height=450',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('path')
 
         webpage = self._download_webpage(url, video_id)
 
-        video_link = self._search_regex(
-            r'(<a.*?class="videoplayer">)', webpage, 'video link')
+        video_url = self._search_regex(
+            r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"',
+            webpage, 'video url')
 
-        url = self._search_regex(r'href="([^"]+)"', video_link, 'video url')
-        width = int_or_none(self._search_regex(
-            r'width:(\d+)px', video_link, 'width', default=None, fatal=False))
-        height = int_or_none(self._search_regex(
-            r'height:(\d+)px', video_link, 'height', default=None, fatal=False))
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
         duration = int_or_none(self._og_search_property(
-            'video:duration', webpage, 'duration'))
+            'video:duration', webpage, 'duration', default=None))
+
         return {
             'id': video_id,
-            'url': url,
-            'width': width,
-            'height': height,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'url': video_url,
+            'title': title,
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
         }

From b859971873915df55668a59a18ccfd259c20800e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 18:15:30 +0600
Subject: [PATCH 0873/1515] [extractor/generic] Rename tvc embed url variable

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6be9e6329..357d58cea 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1312,9 +1312,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(rutv_url, 'RUTV')
 
         # Look for embedded TVC player
-        rutv_url = TVCIE._extract_url(webpage)
-        if rutv_url:
-            return self.url_result(rutv_url, 'TVC')
+        tvc_url = TVCIE._extract_url(webpage)
+        if tvc_url:
+            return self.url_result(tvc_url, 'TVC')
 
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)

From 9872d3110c0d3027dac856e005299f3682ef23ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 18:37:09 +0600
Subject: [PATCH 0874/1515] [extractor/generic] Add support for tvigle embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 357d58cea..40d869c53 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1321,6 +1321,12 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        # Look for embedded Tvigle player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Tvigle')
+
         # Look for embedded TED player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)

From d22dec74ffa2a53a1c04770af37d39f384f3d56c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 19:20:12 +0600
Subject: [PATCH 0875/1515] Add `--force-generic-extractor`

For some extractors that are hard to workout a good _VALID_URL we use very vague and unrestrictive ones,
e.g. just allowing anything after hostname and capturing part of URL as id.
If some of these extractors happen to have an video embed of some different hoster or platform
and this scenario was not handled in extractor itself we end up with inability to download this embed
until extractor is fixed to support embed of this kind.
Forcing downloader to use the generic extractor can be a neat temporary solution for this problem.
Example: FiveTV extractor with Tvigle embed - http://www.5-tv.ru/rabota/broadcasts/48/
---
 youtube_dl/YoutubeDL.py         | 6 ++++++
 youtube_dl/__init__.py          | 1 +
 youtube_dl/extractor/generic.py | 4 +++-
 youtube_dl/options.py           | 4 ++++
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..4b801a917 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -139,6 +139,7 @@ class YoutubeDL(object):
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
+    force_generic_extractor: Force downloader to use the generic extractor
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
@@ -282,6 +283,7 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
+        self._force_generic_extractor_required = params.get('force_generic_extractor', False)
         self.params = params
         self.cache = Cache(self)
 
@@ -633,6 +635,10 @@ class YoutubeDL(object):
         extra_info is a dict containing the extra values to add to each result
         '''
 
+        if not ie_key and self._force_generic_extractor_required:
+            self._force_generic_extractor_required = False
+            ie_key = 'Generic'
+
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]
         else:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ace17857c..215b616de 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -293,6 +293,7 @@ def _real_main(argv=None):
         'autonumber_size': opts.autonumber_size,
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
+        'force_generic_extractor': opts.force_generic_extractor,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
         'retries': opts_retries,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40d869c53..3d672197c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -977,7 +977,9 @@ class GenericIE(InfoExtractor):
                 'upload_date': upload_date,
             }
 
-        if not self._downloader.params.get('test', False) and not is_intentional:
+        if (not self._downloader.params.get('test', False) and
+                not is_intentional and
+                not self._downloader.params.get('force_generic_extractor', False)):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if not full_response:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..096ab6137 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -150,6 +150,10 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
+    general.add_option(
+        '--force-generic-extractor',
+        action='store_true', dest='force_generic_extractor', default=False,
+        help='Force extraction to use the generic extractor')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',

From 3d535e047162af021b3df6086f9a90d0cb0b6100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 19:31:52 +0600
Subject: [PATCH 0876/1515] [tvc] Fix embed regex

---
 youtube_dl/extractor/tvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 6b5d80aee..3a4f393fc 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -27,7 +27,7 @@ class TVCIE(InfoExtractor):
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http:)?//(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 185dbc49749ca81fbb0a61a78c6dd35f2c32b15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:13:14 +0600
Subject: [PATCH 0877/1515] [prosiebensat1] Fix rtmp extraction (Closes #5962)

---
 youtube_dl/extractor/prosiebensat1.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 255d4abc1..6b13eb605 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -206,8 +206,8 @@ class ProSiebenSat1IE(InfoExtractor):
     def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
-        access_token = 'testclient'
-        client_name = 'kolibri-1.2.5'
+        access_token = 'prosieben'
+        client_name = 'kolibri-1.12.6'
         client_location = url
 
         videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
@@ -275,13 +275,17 @@ class ProSiebenSat1IE(InfoExtractor):
         for source in urls_sources:
             protocol = source['protocol']
             if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', source['url'])
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source['url'])
                 if not mobj:
                     continue
+                path = mobj.group('path')
+                mp4colon_index = path.rfind('mp4:')
+                app = path[:mp4colon_index]
+                play_path = path[mp4colon_index:]
                 formats.append({
-                    'url': mobj.group('url'),
-                    'app': mobj.group('app'),
-                    'play_path': mobj.group('playpath'),
+                    'url': '%s/%s' % (mobj.group('url'), app),
+                    'app': app,
+                    'play_path': play_path,
                     'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
                     'page_url': 'http://www.prosieben.de',
                     'vbr': fix_bitrate(source['bitrate']),

From 8b6c896c4b60fe13b30227071aba2783222132a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:18:13 +0600
Subject: [PATCH 0878/1515] [prosiebensat1] Add title regex

---
 youtube_dl/extractor/prosiebensat1.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 6b13eb605..536a42dc8 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -177,6 +177,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<header class="clearfix">\s*<h3>(.+?)</h3>',
         r'<!-- start video -->\s*<h1>(.+?)</h1>',
         r'<h1 class="att-name">\s*(.+?)</h1>',
+        r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',

From 9f4323252abade4f10b0884682f92cedc78b4d4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:56:50 +0600
Subject: [PATCH 0879/1515] [YoutubeDL] Fix for multiple URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4b801a917..8dbad7cf8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -283,7 +283,6 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
-        self._force_generic_extractor_required = params.get('force_generic_extractor', False)
         self.params = params
         self.cache = Cache(self)
 
@@ -1504,6 +1503,7 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
+                self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
                 # It also downloads the videos
                 res = self.extract_info(url)
             except UnavailableVideoError:

From 61aa5ba36eea3b7cf8c3570ab33604dd2c13b855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 02:05:21 +0600
Subject: [PATCH 0880/1515] [YoutubeDL] Remove global state for
 force_generic_extractor flag in favor of passing argument

---
 youtube_dl/YoutubeDL.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8dbad7cf8..dd2d8cb3c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -627,15 +627,14 @@ class YoutubeDL(object):
             info_dict.setdefault(key, value)
 
     def extract_info(self, url, download=True, ie_key=None, extra_info={},
-                     process=True):
+                     process=True, force_generic_extractor=False):
         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result
         '''
 
-        if not ie_key and self._force_generic_extractor_required:
-            self._force_generic_extractor_required = False
+        if not ie_key and force_generic_extractor:
             ie_key = 'Generic'
 
         if ie_key:
@@ -663,7 +662,7 @@ class YoutubeDL(object):
                     }
                 self.add_default_extra_info(ie_result, ie, url)
                 if process:
-                    return self.process_ie_result(ie_result, download, extra_info)
+                    return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
@@ -688,7 +687,7 @@ class YoutubeDL(object):
             'extractor_key': ie.ie_key(),
         })
 
-    def process_ie_result(self, ie_result, download=True, extra_info={}):
+    def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -716,7 +715,8 @@ class YoutubeDL(object):
             return self.extract_info(ie_result['url'],
                                      download,
                                      ie_key=ie_result.get('ie_key'),
-                                     extra_info=extra_info)
+                                     extra_info=extra_info,
+                                     force_generic_extractor=force_generic_extractor)
         elif result_type == 'url_transparent':
             # Use the information from the embedding page
             info = self.extract_info(
@@ -1503,9 +1503,9 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
-                self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
                 # It also downloads the videos
-                res = self.extract_info(url)
+                res = self.extract_info(
+                    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
             except UnavailableVideoError:
                 self.report_error('unable to download video')
             except MaxDownloadsReached:

From 0072afca8e02052c77dc3b7009e51114887e31b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 02:21:29 +0600
Subject: [PATCH 0881/1515] [YoutubeDL] Remove force_generic_extractor arg from
 process_ie_result

---
 youtube_dl/YoutubeDL.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dd2d8cb3c..a7d3a1c01 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -662,7 +662,7 @@ class YoutubeDL(object):
                     }
                 self.add_default_extra_info(ie_result, ie, url)
                 if process:
-                    return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
+                    return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
@@ -687,7 +687,7 @@ class YoutubeDL(object):
             'extractor_key': ie.ie_key(),
         })
 
-    def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
+    def process_ie_result(self, ie_result, download=True, extra_info={}):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -715,8 +715,7 @@ class YoutubeDL(object):
             return self.extract_info(ie_result['url'],
                                      download,
                                      ie_key=ie_result.get('ie_key'),
-                                     extra_info=extra_info,
-                                     force_generic_extractor=force_generic_extractor)
+                                     extra_info=extra_info)
         elif result_type == 'url_transparent':
             # Use the information from the embedding page
             info = self.extract_info(

From 4f3bf679f5a764f7a26d3c45c82be43e34a3cc4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:09:35 +0600
Subject: [PATCH 0882/1515] [vk] Fix authentication for non-ASCII
 login/password

---
 youtube_dl/extractor/vk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index cc384adbf..d0e772108 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -119,8 +119,8 @@ class VKIE(InfoExtractor):
             'act': 'login',
             'role': 'al_frame',
             'expire': '1',
-            'email': username,
-            'pass': password,
+            'email': username.encode('cp1251'),
+            'pass': password.encode('cp1251'),
         }
 
         request = compat_urllib_request.Request('https://login.vk.com/?act=login',

From 9fcbd5db2abb5a56beefe4a64486da692705ad12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:24:36 +0600
Subject: [PATCH 0883/1515] [pornhub] Add support for embeds

---
 youtube_dl/extractor/pornhub.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index daa284ea2..3c99b4def 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -19,7 +19,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-f]+)'
     _TEST = {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
@@ -39,7 +39,8 @@ class PornHubIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = compat_urllib_request.Request(
+            'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 65d161c480e9964026e618a2e95f9fc9eb8119e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:36:16 +0600
Subject: [PATCH 0884/1515] [extractor/generic] Add support for pornhub embeds

---
 youtube_dl/extractor/generic.py | 5 +++++
 youtube_dl/extractor/pornhub.py | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40d869c53..f683760e4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -42,6 +42,7 @@ from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
+from .pornhub import PornHubIE
 
 
 class GenericIE(InfoExtractor):
@@ -1321,6 +1322,10 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        pornhub_url = PornHubIE._extract_url(webpage)
+        if pornhub_url:
+            return self.url_result(pornhub_url, 'PornHub')
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3c99b4def..8565d7551 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -32,6 +32,13 @@ class PornHubIE(InfoExtractor):
         }
     }
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?pornhub\.com/embed/\d+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _extract_count(self, pattern, webpage, name):
         return str_to_int(self._search_regex(
             pattern, webpage, '%s count' % name, fatal=False))

From 78e2b74bb945dc7f1724f7486405dd523486d634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:39:14 +0600
Subject: [PATCH 0885/1515] [tumblr] Add support for pornhub embeds (Closes
 #5963)

---
 youtube_dl/extractor/generic.py | 1 +
 youtube_dl/extractor/tumblr.py  | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f683760e4..f6b984300 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1322,6 +1322,7 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        # Look for embedded PornHub player
         pornhub_url = PornHubIE._extract_url(webpage)
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index e6218808f..63c20310d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .pornhub import PornHubIE
 
 
 class TumblrIE(InfoExtractor):
@@ -55,6 +56,10 @@ class TumblrIE(InfoExtractor):
         if vid_me_embed_url is not None:
             return self.url_result(vid_me_embed_url, 'Vidme')
 
+        pornhub_url = PornHubIE._extract_url(webpage)
+        if pornhub_url:
+            return self.url_result(pornhub_url, 'PornHub')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From b4e1576aee7cf18f5842714c87985ae0b72f1546 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 13 Jun 2015 06:09:44 -0500
Subject: [PATCH 0886/1515] Brightcove extractor: support
 customBC.createVideo(...); method found in
 http://www.americanbar.org/groups/family_law.html and
 http://america.aljazeera.com/watch/shows/america-tonight/2015/6/exclusive-hunting-isil-with-the-pkk.html

---
 youtube_dl/extractor/brightcove.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c1d4320e1..20a6ed965 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -188,7 +188,19 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
+        if matches:
+            return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
+
+        custombcs = re.findall(r'customBC.\createVideo\((.+?)\);',webpage)
+        if custombcs:
+            urls = []
+            for match in custombcs:
+                # brightcove playerkey begins with AQ and is 50 characters in length,
+                # however it's appended to itself in places, so truncate.
+                f = re.search(r'["\'](AQ[^"\']{48}).*?["\'](\d+)["\']', match)
+                if f:
+                    urls.append('brightcove:playerKey='+f.group(1)+'&%40videoPlayer='+f.group(2))
+            return urls
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From af9cdee9cba610aa3924f90a8a3fcd7dd43c65eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 19:53:32 +0600
Subject: [PATCH 0887/1515] [brightcove] Improve and generalize brightcove URL
 extraction from JS

---
 youtube_dl/extractor/brightcove.py | 35 +++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 20a6ed965..d768f99e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -156,6 +156,28 @@ class BrightcoveIE(InfoExtractor):
         linkBase = find_param('linkBaseURL')
         if linkBase is not None:
             params['linkBaseURL'] = linkBase
+        return cls._make_brightcove_url(params)
+
+    @classmethod
+    def _build_brighcove_url_from_js(cls, object_js):
+        # The layout of JS is as follows:
+        # customBC.createVideo = function (width, height, playerID, playerKey, videoPlayer, VideoRandomID) {
+        #   // build Brightcove <object /> XML
+        # }
+        m = re.search(
+            r'''(?x)customBC.\createVideo\(
+                .*?                                                  # skipping width and height
+                ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
+                ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters
+                                                                     # in length, however it's appended to itself
+                                                                     # in places, so truncate
+                ["\'](?P<videoID>\d+)["\']                           # @videoPlayer
+            ''', object_js)
+        if m:
+            return cls._make_brightcove_url(m.groupdict())
+
+    @classmethod
+    def _make_brightcove_url(cls, params):
         data = compat_urllib_parse.urlencode(params)
         return cls._FEDERATED_URL_TEMPLATE % data
 
@@ -191,16 +213,9 @@ class BrightcoveIE(InfoExtractor):
         if matches:
             return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
-        custombcs = re.findall(r'customBC.\createVideo\((.+?)\);',webpage)
-        if custombcs:
-            urls = []
-            for match in custombcs:
-                # brightcove playerkey begins with AQ and is 50 characters in length,
-                # however it's appended to itself in places, so truncate.
-                f = re.search(r'["\'](AQ[^"\']{48}).*?["\'](\d+)["\']', match)
-                if f:
-                    urls.append('brightcove:playerKey='+f.group(1)+'&%40videoPlayer='+f.group(2))
-            return urls
+        return list(filter(None, [
+            cls._build_brighcove_url_from_js(custom_bc)
+            for custom_bc in re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)]))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From 0029071adbdc0e1469d76cdc7e058c2f77299610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 07:43:14 +0600
Subject: [PATCH 0888/1515] [dramefever] Improve and simplify

---
 youtube_dl/extractor/dramafever.py | 172 +++++++++++++++++------------
 1 file changed, 101 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 40787ffcd..0f33a61a3 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -1,104 +1,111 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
+import itertools
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_HTTPError,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class DramaFeverIE(InfoExtractor):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)/'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)'
+    _TEST = {
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
             'ext': 'flv',
             'title': 'Cooking with Shin 4512.1',
+            'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1404336058,
             'upload_date': '20140702',
-            'description': 'Served at all special occasions and featured in the hit drama Heirs, Shin cooks Red Bean Rice.',
+            'duration': 343,
         }
-    }]
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url).replace("/", ".")
+        video_id = self._match_id(url).replace('/', '.')
 
-        consumer_secret = self._get_consumer_secret(video_id)
+        try:
+            feed = self._download_json(
+                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
+                video_id, 'Downloading episode JSON')['channel']['item']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                raise ExtractorError(
+                    'Currently unavailable in your country.', expected=True)
+            raise
 
-        ep_json = self._download_json(
-            "http://www.dramafever.com/amp/episode/feed.json?guid=%s" % video_id,
-            video_id, note='Downloading episode metadata',
-            errnote="Video may not be available for your location")["channel"]["item"]
-
-        title = ep_json["media-group"]["media-title"]
-        description = ep_json["media-group"]["media-description"]
-        thumbnail = ep_json["media-group"]["media-thumbnail"]["@attributes"]["url"]
-        duration = int(ep_json["media-group"]["media-content"][0]["@attributes"]["duration"])
-        mobj = re.match(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", ep_json["pubDate"])
-        upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) if mobj is not None else None
+        media_group = feed.get('media-group', {})
 
         formats = []
-        for vid_format in ep_json["media-group"]["media-content"]:
-            src = vid_format["@attributes"]["url"]
-            if '.f4m' in src:
-                formats.extend(self._extract_f4m_formats(src, video_id))
-
+        for media_content in media_group['media-content']:
+            src = media_content.get('@attributes', {}).get('url')
+            if not src:
+                continue
+            ext = determine_ext(src)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    src, video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': src,
+                })
         self._sort_formats(formats)
-        video_subtitles = self.extract_subtitles(video_id, consumer_secret)
+
+        title = media_group.get('media-title')
+        description = media_group.get('media-description')
+        duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
+        thumbnail = self._proto_relative_url(
+            media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
+        timestamp = parse_iso8601(feed.get('pubDate'), ' ')
+
+        subtitles = {}
+        for media_subtitle in media_group.get('media-subTitle', []):
+            lang = media_subtitle.get('@attributes', {}).get('lang')
+            href = media_subtitle.get('@attributes', {}).get('href')
+            if not lang or not href:
+                continue
+            subtitles[lang] = [{
+                'ext': 'ttml',
+                'url': href,
+            }]
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
-    def _get_consumer_secret(self, video_id):
-        df_js = self._download_webpage(
-            "http://www.dramafever.com/static/126960d/v2/js/plugins/jquery.threadedcomments.js", video_id)
-        return self._search_regex(r"'cs': '([0-9a-zA-Z]+)'", df_js, "cs")
 
-    def _get_episodes(self, series_id, consumer_secret, episode_filter=None):
-        _PAGE_SIZE = 60
-
-        curr_page = 1
-        max_pages = curr_page + 1
-        results = []
-        while max_pages >= curr_page:
-            page_url = "http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d" % \
-                       (consumer_secret, series_id, _PAGE_SIZE, curr_page)
-            series = self._download_json(
-                page_url, series_id, note="Downloading series json page #%d" % curr_page)
-            max_pages = series['num_pages']
-            results.extend([ep for ep in series['value'] if episode_filter is None or episode_filter(ep)])
-            curr_page += 1
-        return results
-
-    def _get_subtitles(self, video_id, consumer_secret):
-
-        res = None
-        info = self._get_episodes(
-            video_id.split(".")[0], consumer_secret,
-            episode_filter=lambda x: x['guid'] == video_id)
-
-        if len(info) == 1 and info[0]['subfile'] != '':
-            res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
-        return res
-
-
-class DramaFeverSeriesIE(DramaFeverIE):
+class DramaFeverSeriesIE(InfoExtractor):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)/\d*[a-zA-Z_][a-zA-Z0-9_]*/'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512',
             'title': 'Cooking with Shin',
-            'description': 'Professional chef and cooking instructor Shin Kim takes some of the delicious dishes featured in your favorite dramas and shows you how to make them right at home.',
+            'description': 'md5:84a3f26e3cdc3fb7f500211b3593b5c1',
         },
         'playlist_count': 4,
     }, {
@@ -106,25 +113,48 @@ class DramaFeverSeriesIE(DramaFeverIE):
         'info_dict': {
             'id': '124',
             'title': 'IRIS',
-            'description': 'Lee Byung Hun and Kim Tae Hee star in this powerhouse drama and ratings megahit of action, intrigue and romance.',
+            'description': 'md5:b3a30e587cf20c59bd1c01ec0ee1b862',
         },
         'playlist_count': 20,
     }]
 
+    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
+    _PAGE_SIZE = 5  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
+
+    def _get_consumer_secret(self, video_id):
+        mainjs = self._download_webpage(
+            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
+            video_id, 'Downloading main.js', fatal=False)
+        if not mainjs:
+            return self._CONSUMER_SECRET
+        return self._search_regex(
+            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
+            'consumer secret', default=self._CONSUMER_SECRET)
+
     def _real_extract(self, url):
         series_id = self._match_id(url)
+
         consumer_secret = self._get_consumer_secret(series_id)
 
-        series_json = self._download_json(
-            "http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s" % (consumer_secret, series_id),
-            series_id, note='Downloading series metadata')["series"][series_id]
+        series = self._download_json(
+            'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
+            % (consumer_secret, series_id),
+            series_id, 'Downloading series JSON')['series'][series_id]
 
-        title = series_json["name"]
-        description = series_json["description_short"]
+        title = clean_html(series['name'])
+        description = clean_html(series.get('description') or series.get('description_short'))
 
-        episodes = self._get_episodes(series_id, consumer_secret)
         entries = []
-        for ep in episodes:
-            entries.append(self.url_result(
-                'http://www.dramafever.com%s' % ep['episode_url'], 'DramaFever', ep['guid']))
+        for page_num in itertools.count(1):
+            episodes = self._download_json(
+                'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
+                % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
+                series_id, 'Downloading episodes JSON page #%d' % page_num)
+            for episode in episodes.get('value', []):
+                entries.append(self.url_result(
+                    compat_urlparse.urljoin(url, episode['episode_url']),
+                    'DramaFever', episode.get('guid')))
+            if page_num == episodes['num_pages']:
+                break
+
         return self.playlist_result(entries, series_id, title, description)

From 70a2002399b46aa0cde2879d856d1bb68e2c6f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:50:23 +0600
Subject: [PATCH 0889/1515] [dramafever:series] Fix _VALID_URL (Closes #5973)

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 0f33a61a3..42e0df24e 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -99,7 +99,7 @@ class DramaFeverIE(InfoExtractor):
 
 class DramaFeverSeriesIE(InfoExtractor):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {

From 463b2e5542a85d5cd41b255a71833fec7b4f51e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:51:07 +0600
Subject: [PATCH 0890/1515] [dramafever:series] Rollback _PAGE_SIZE to max
 possible

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 42e0df24e..c4b7c0b68 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -119,7 +119,7 @@ class DramaFeverSeriesIE(InfoExtractor):
     }]
 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
-    _PAGE_SIZE = 5  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
+    _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 
     def _get_consumer_secret(self, video_id):
         mainjs = self._download_webpage(

From 450d89ddc12d80a500a2429632d35a0045cf630b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:58:26 +0600
Subject: [PATCH 0891/1515] [dramafever] Improve _VALID_URL

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index c4b7c0b68..a34aad486 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 class DramaFeverIE(InfoExtractor):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TEST = {
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {

From 976b03c56bddf20c978820474e307457523f4c05 Mon Sep 17 00:00:00 2001
From: chaoskagami <chaos.kagami@gmail.com>
Date: Sun, 14 Jun 2015 00:18:40 -0400
Subject: [PATCH 0892/1515] Quality note for niconico - at least notify whether
 you'll get low or src

---
 youtube_dl/extractor/niconico.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 3cecebf95..e10348004 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -184,6 +184,11 @@ class NiconicoIE(InfoExtractor):
             extension = determine_ext(video_real_url)
         video_format = extension.upper()
 
+        if video_real_url.endswith('low'):
+            format_note = 'low'
+        else:
+            format_note = 'src'
+
         thumbnail = (
             xpath_text(video_info, './/thumbnail_url') or
             self._html_search_meta('image', webpage, 'thumbnail', default=None) or
@@ -242,6 +247,7 @@ class NiconicoIE(InfoExtractor):
             'title': title,
             'ext': extension,
             'format': video_format,
+            'format_note' : format_note,
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 180940e02df60129bce36035b4a2fd79c0c60995 Mon Sep 17 00:00:00 2001
From: Shrimadhav U K <SpEcHiDe@users.noreply.github.com>
Date: Sun, 14 Jun 2015 11:19:42 +0530
Subject: [PATCH 0893/1515] spelling mistake corrected

acces changed to accessing
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..aacec2958 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -119,7 +119,7 @@ class YoutubeDL(object):
 
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
-    videopassword:     Password for acces a video.
+    videopassword:     Password for accessing a video.
     usenetrc:          Use netrc for authentication instead.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.

From 755a9d3d1a8f99b061c8d29525d629b8ad6061a4 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Jun 2015 20:58:15 +0300
Subject: [PATCH 0894/1515] [tvplay] Add support for NovaTv

---
 youtube_dl/extractor/tvplay.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index e83e31a31..79863e781 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -26,6 +26,7 @@ class TVPlayIE(InfoExtractor):
            viasat4play\.no/programmer|
            tv6play\.no/programmer|
            tv3play\.dk/programmer|
+           play\.novatv\.bg/programi
         )/[^/]+/(?P<id>\d+)
         '''
     _TESTS = [
@@ -173,6 +174,22 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
+            'info_dict': {
+                'id': '624952',
+                'ext': 'flv',
+                'title': 'Здравей, България (12.06.2015 г.) ',
+                'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
+                'duration': 8838,
+                'timestamp': 1434100372,
+                'upload_date': '20150612',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):

From 9fd24e3a227a059eed07b679dac858e5bd747123 Mon Sep 17 00:00:00 2001
From: jomo <github@jomo.tv>
Date: Sun, 14 Jun 2015 21:50:03 +0200
Subject: [PATCH 0895/1515] LiveLeak: support more original videos

some (old?) videos use ...mp4.h264_270p.mp4... instead of ...mp4.h264_base.mp4...
This is an addition to #4768
---
 youtube_dl/extractor/liveleak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 35822067f..431f2e85d 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -85,7 +85,7 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
-            orig_url = s['file'].replace('.h264_base.mp4', '')
+            orig_url = re.sub(r'.h264_.+\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({
                     'format_id': 'original-%s' % i,

From 8f75761f24f9f2599efe100b5a094182af6403d0 Mon Sep 17 00:00:00 2001
From: jomo <github@jomo.tv>
Date: Sun, 14 Jun 2015 22:41:44 +0200
Subject: [PATCH 0896/1515] LiveLak: add test for URLs with 'h264_270p'

---
 youtube_dl/extractor/liveleak.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 431f2e85d..c658cc92b 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -40,6 +40,16 @@ class LiveLeakIE(InfoExtractor):
             'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
             'age_limit': 18,
         }
+    }, {
+        'url': 'http://www.liveleak.com/view?i=801_1409392012',
+        'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
+        'info_dict': {
+            'id': '801_1409392012',
+            'ext': 'mp4',
+            'description': "Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.",
+            'uploader': 'bony333',
+            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia'
+        }
     }]
 
     def _real_extract(self, url):

From 00ac23e6e06bf6de59d5d5b3f42ff64ce039fee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:51:21 +0600
Subject: [PATCH 0897/1515] [liveleak] Improve regex for restoring original
 video URL

---
 youtube_dl/extractor/liveleak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c658cc92b..e82f21ea7 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -95,7 +95,7 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
-            orig_url = re.sub(r'.h264_.+\.mp4', '', s['file'])
+            orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({
                     'format_id': 'original-%s' % i,

From afa1ded425ffe12a5b90bcd4d316c43941a5dc1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:54:05 +0600
Subject: [PATCH 0898/1515] [liveleak] Clarify rationale for restoring raw
 video

---
 youtube_dl/extractor/liveleak.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index e82f21ea7..0a4e473d6 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -95,6 +95,9 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
+            # Removing '.h264_*.mp4' gives the raw video, which is essentially
+            # the same video without the LiveLeak logo at the top (see
+            # https://github.com/rg3/youtube-dl/pull/4768)
             orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({

From b95cfa917025750805fb873fc4e2eb161241b22b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:54:49 +0600
Subject: [PATCH 0899/1515] [liveleak] Clarify test

---
 youtube_dl/extractor/liveleak.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 0a4e473d6..857edfde2 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -41,6 +41,7 @@ class LiveLeakIE(InfoExtractor):
             'age_limit': 18,
         }
     }, {
+        # Covers https://github.com/rg3/youtube-dl/pull/5983
         'url': 'http://www.liveleak.com/view?i=801_1409392012',
         'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
         'info_dict': {

From 5774ef35c4d167f7c959041bf4efc5581a98f0a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:57:07 +0600
Subject: [PATCH 0900/1515] [options] Add missing whitespace for --fixup
 description

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..740458e51 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -725,7 +725,7 @@ def parseOpts(overrideArguments=None):
         metavar='POLICY', dest='fixup', default='detect_or_warn',
         help='Automatically correct known faults of the file. '
              'One of never (do nothing), warn (only emit a warning), '
-             'detect_or_warn(the default; fix file if we can, warn otherwise)')
+             'detect_or_warn (the default; fix file if we can, warn otherwise)')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',

From 67d95f177c7ffedfc8f8b086535013a1a7a48b29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 03:43:33 +0600
Subject: [PATCH 0901/1515] [niconico] Simplify format info

---
 youtube_dl/extractor/niconico.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index e10348004..0f8aa5ada 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -182,12 +182,6 @@ class NiconicoIE(InfoExtractor):
         extension = xpath_text(video_info, './/movie_type')
         if not extension:
             extension = determine_ext(video_real_url)
-        video_format = extension.upper()
-
-        if video_real_url.endswith('low'):
-            format_note = 'low'
-        else:
-            format_note = 'src'
 
         thumbnail = (
             xpath_text(video_info, './/thumbnail_url') or
@@ -246,8 +240,7 @@ class NiconicoIE(InfoExtractor):
             'url': video_real_url,
             'title': title,
             'ext': extension,
-            'format': video_format,
-            'format_note' : format_note,
+            'format_id': 'economy' if video_real_url.endswith('low') else 'normal',
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 2a0fcf6113c3f0c8d0510167fd7017cc0fdfa622 Mon Sep 17 00:00:00 2001
From: zx8 <zx8@mac>
Date: Mon, 15 Jun 2015 00:27:43 +0100
Subject: [PATCH 0902/1515] [safari] make url regex more lenient

---
 youtube_dl/extractor/safari.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 10251f29e..20ba6fa33 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -83,7 +83,7 @@ class SafariIE(SafariBaseIE):
                                     library/view/[^/]+|
                                     api/v1/book
                                 )/
-                                (?P<course_id>\d+)/
+                                (?P<course_id>[^/]+)/
                                     (?:chapter(?:-content)?/)?
                                 (?P<part>part\d+)\.html
     '''
@@ -122,7 +122,7 @@ class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
 
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>\d+)/?(?:[#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>[^/]+)/?(?:[#?]|$)'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',

From 4b9f9010b0d744969189c383e98e8729f9fe9623 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Jun 2015 01:35:50 +0200
Subject: [PATCH 0903/1515] release 2015.06.15

---
 README.md              | 4 ++--
 docs/supportedsites.md | 7 +++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index f3d83c89f..5f3a08f5a 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ which means you can modify it, redistribute it or use it however you like.
     -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
     --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
     --dump-user-agent                Display the current browser identification
-    --list-extractors                List all supported extractors and the URLs they would handle
+    --list-extractors                List all supported extractors
     --extractor-descriptions         Output descriptions of all supported extractors
     --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
@@ -223,7 +223,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
+    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the default;
                                      fix file if we can, warn otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d147b53fe..220e52b98 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -120,6 +120,8 @@
  - **divxstage**: DivxStage
  - **Dotsub**
  - **DouyuTV**
+ - **dramafever**
+ - **dramafever:series**
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
@@ -153,6 +155,7 @@
  - **fernsehkritik.tv**
  - **fernsehkritik.tv:postecke**
  - **Firstpost**
+ - **FiveTV**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
@@ -217,6 +220,7 @@
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
  - **IPrima**
+ - **iqiyi**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -407,6 +411,7 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **Ruutu**
  - **safari**: safaribooksonline.com online video
  - **safari:course**: safaribooksonline.com online courses
  - **Sandia**: Sandia National Laboratories
@@ -519,6 +524,8 @@
  - **TV2**
  - **TV2Article**
  - **TV4**: tv4.se and tv4play.se
+ - **TVC**
+ - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
  - **tvp.pl:Series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9cf84ff71..34a13cb81 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.04.1'
+__version__ = '2015.06.15'

From 4af98ecdfb896c16e73cb9f7306908cc686782e8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 18:49:27 +0800
Subject: [PATCH 0904/1515] [vbox7] Fix extraction (fixes #5967)

---
 youtube_dl/extractor/vbox7.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index dd026748d..722eb5236 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -26,11 +27,21 @@ class Vbox7IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        redirect_page, urlh = self._download_webpage_handle(url, video_id)
-        new_location = self._search_regex(r'window\.location = \'(.*)\';',
-                                          redirect_page, 'redirect location')
-        redirect_url = urlh.geturl() + new_location
-        webpage = self._download_webpage(redirect_url, video_id,
+        # need to get the page 3 times for the correct jsSecretToken cookie
+        # which is necessary for the correct title
+        def get_session_id():
+            redirect_page = self._download_webpage(url, video_id)
+            session_id_url = self._search_regex(
+                r'var\s*url\s*=\s*\'([^\']+)\';', redirect_page,
+                'session id url')
+            self._download_webpage(
+                compat_urlparse.urljoin(url, session_id_url), video_id,
+                'Getting session id')
+
+        get_session_id()
+        get_session_id()
+
+        webpage = self._download_webpage(url, video_id,
                                          'Downloading redirect page')
 
         title = self._html_search_regex(r'<title>(.*)</title>',

From aed473ccf9d9da73b1b80ee8b06d00ee66a3769d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 22:41:24 +0800
Subject: [PATCH 0905/1515] [youku] PEP8

---
 youtube_dl/extractor/youku.py | 80 +++++++++++++++++------------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index aed6b960a..4e47fca8a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -11,6 +11,7 @@ from ..compat import compat_urllib_parse
 
 bytes_is_str = (bytes == str)  # for compatible
 
+
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
@@ -21,13 +22,13 @@ class YoukuIE(InfoExtractor):
     '''
 
     _TEST = {
-            'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
-            'md5': '5f3af4192eabacc4501508d54a8cabd7',
-            'info_dict': {
-                'id': 'XMTc1ODE5Njcy',
-                'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
-                'ext': 'flv'
-            }
+        'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
+        'md5': '5f3af4192eabacc4501508d54a8cabd7',
+        'info_dict': {
+            'id': 'XMTc1ODE5Njcy',
+            'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
+            'ext': 'flv'
+        }
     }
 
     def construct_video_urls(self, data1, data2):
@@ -36,7 +37,7 @@ class YoukuIE(InfoExtractor):
             ls = list(range(256))
             t = 0
             for i in range(256):
-                t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
+                t = (t + ls[i] + ord(s1[i % len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
             s = '' if not bytes_is_str else b''
             x, y = 0, 0
@@ -45,16 +46,16 @@ class YoukuIE(InfoExtractor):
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
                 if isinstance(s2[i], int):
-                    s += chr(s2[i] ^ ls[(ls[x]+ls[y]) % 256])
+                    s += chr(s2[i] ^ ls[(ls[x] + ls[y]) % 256])
                 else:
-                    s += chr(ord(s2[i]) ^ ls[(ls[x]+ls[y]) % 256])
+                    s += chr(ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
             return s
 
         sid, token = yk_t(
             'becaf9be',
-            base64.b64decode(bytes(data2['ep'], 'ascii')) \
-                if not bytes_is_str \
-                else base64.b64decode(data2['ep'])
+            base64.b64decode(bytes(data2['ep'], 'ascii'))
+            if not bytes_is_str
+            else base64.b64decode(data2['ep'])
         ).split('_')
 
         # get oip
@@ -89,13 +90,13 @@ class YoukuIE(InfoExtractor):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
                 'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
-                if not bytes_is_str \
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii')
+                if not bytes_is_str
                 else ('%s_%s_%s' % (sid, fileid, token))
             )
             ep = base64.b64encode(
-                bytes(ep_t, 'latin') \
-                if not bytes_is_str \
+                bytes(ep_t, 'latin')
+                if not bytes_is_str
                 else ep_t
             ).decode()
             return ep
@@ -121,9 +122,9 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_' + str(int(n)+1).zfill(2) + \
+                    '_' + str(int(n) + 1).zfill(2) + \
                     '/st/' + self.parse_ext_l(format) + \
-                    '/fileid/' + get_fileid(format, n)  + '?' + \
+                    '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
@@ -132,34 +133,34 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv'   : '0',
-            'mp4'   : '1',
-            'hd2'   : '2',
-            'hd3'   : '3',
-            '3gp'   : '0',
-            '3gphd' : '1'
+            'flv': '0',
+            'mp4': '1',
+            'hd2': '2',
+            'hd3': '3',
+            '3gp': '0',
+            '3gphd': '1'
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
-            'flv'   : 'flv',
-            'mp4'   : 'mp4',
-            'hd2'   : 'flv',
-            'hd3'   : 'flv',
-            '3gp'   : 'flv',
-            '3gphd' : 'mp4'
+            'flv': 'flv',
+            'mp4': 'mp4',
+            'hd2': 'flv',
+            'hd3': 'flv',
+            '3gp': 'flv',
+            '3gphd': 'mp4'
         }
         return ext_dict[fm]
 
     def get_format_name(self, fm):
         _dict = {
-            '3gp'   : 'h6',
-            '3gphd' : 'h5',
-            'flv'   : 'h4',
-            'mp4'   : 'h3',
-            'hd2'   : 'h2',
-            'hd3'   : 'h1'
+            '3gp': 'h6',
+            '3gphd': 'h5',
+            'flv': 'h4',
+            'mp4': 'h3',
+            'hd2': 'h2',
+            'hd3': 'h1'
         }
         return _dict[fm]
 
@@ -194,10 +195,9 @@ class YoukuIE(InfoExtractor):
         # construct info
         entries = []
         for fm in data1['streamtypes']:
-            #formats = []
             video_urls = video_urls_dict[fm]
             for i in range(len(video_urls)):
-                if len(entries) < i+1:
+                if len(entries) < i + 1:
                     entries.append({'formats': []})
                 entries[i]['formats'].append(
                     {
@@ -211,7 +211,7 @@ class YoukuIE(InfoExtractor):
         for i in range(len(entries)):
             entries[i].update(
                 {
-                    'id': '_part%d' % (i+1),
+                    'id': '_part%d' % (i + 1),
                     'title': title,
                 }
             )

From 054932f4035d606946f0c054c02cf87496b753f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:46:10 +0600
Subject: [PATCH 0906/1515] [vk] Fix extraction (Closes #5987)

---
 youtube_dl/extractor/vk.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d0e772108..6aeba109d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -175,16 +175,16 @@ class VKIE(InfoExtractor):
                 m_rutube.group(1).replace('\\', ''))
             return self.url_result(rutube_url)
 
-        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.*?});', info_page)
+        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
         if m_opts:
-            m_opts_url = re.search(r"url\s*:\s*'([^']+)", m_opts.group(1))
+            m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))
             if m_opts_url:
                 opts_url = m_opts_url.group(1)
                 if opts_url.startswith('//'):
                     opts_url = 'http:' + opts_url
                 return self.url_result(opts_url)
 
-        data_json = self._search_regex(r'var vars = ({.*?});', info_page, 'vars')
+        data_json = self._search_regex(r'var\s+vars\s*=\s*({.+?});', info_page, 'vars')
         data = json.loads(data_json)
 
         # Extract upload date

From 7c7dd9dc7fe8139196c7bd1c512301a61f9f362b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:47:01 +0600
Subject: [PATCH 0907/1515] [vk] Fix upload date extraction

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 6aeba109d..f974f8fef 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -189,7 +189,7 @@ class VKIE(InfoExtractor):
 
         # Extract upload date
         upload_date = None
-        mobj = re.search(r'id="mv_date_wrap".*?Added ([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
+        mobj = re.search(r'id="mv_date(?:_views)?_wrap"[^>]*>([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
         if mobj is not None:
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))

From 8117df4cd9e49a3c7369db3cab6c0b94365c7786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:55:25 +0600
Subject: [PATCH 0908/1515] [vk] Extract view count

---
 youtube_dl/extractor/vk.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f974f8fef..38ff3c1a9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -13,6 +13,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     orderedSet,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
 )
@@ -34,6 +35,7 @@ class VKIE(InfoExtractor):
                 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                 'duration': 195,
                 'upload_date': '20120212',
+                'view_count': int,
             },
         },
         {
@@ -45,7 +47,8 @@ class VKIE(InfoExtractor):
                 'uploader': 'Tom Cruise',
                 'title': 'No name',
                 'duration': 9,
-                'upload_date': '20130721'
+                'upload_date': '20130721',
+                'view_count': int,
             }
         },
         {
@@ -59,6 +62,7 @@ class VKIE(InfoExtractor):
                 'title': 'Lin Dan',
                 'duration': 101,
                 'upload_date': '20120730',
+                'view_count': int,
             }
         },
         {
@@ -73,7 +77,8 @@ class VKIE(InfoExtractor):
                 'uploader': 'Триллеры',
                 'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
                 'duration': 8352,
-                'upload_date': '20121218'
+                'upload_date': '20121218',
+                'view_count': int,
             },
             'skip': 'Requires vk account credentials',
         },
@@ -100,6 +105,7 @@ class VKIE(InfoExtractor):
                 'title': 'Книга Илая',
                 'duration': 6771,
                 'upload_date': '20140626',
+                'view_count': int,
             },
             'skip': 'Only works from Russia',
         },
@@ -194,6 +200,10 @@ class VKIE(InfoExtractor):
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
+        view_count = str_to_int(self._search_regex(
+            r'"mv_views_count_number"[^>]*>([\d,.]+) views<',
+            info_page, 'view count', fatal=False))
+
         formats = [{
             'format_id': k,
             'url': v,
@@ -210,6 +220,7 @@ class VKIE(InfoExtractor):
             'uploader': data.get('md_author'),
             'duration': data.get('duration'),
             'upload_date': upload_date,
+            'view_count': view_count,
         }
 
 

From 02175a7986c4223a0ed27a872c1ca16926913e05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:01:26 +0600
Subject: [PATCH 0909/1515] [youtube:search] Fix search query (Closes #5988)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3448bec4f..9e2671192 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1504,7 +1504,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
 
         for pagenum in itertools.count(1):
             url_query = {
-                'search_query': query,
+                'search_query': query.encode('utf-8'),
                 'page': pagenum,
                 'spf': 'navigate',
             }

From c203be3fb4f00388c81564dc0c85ff8a10ff4553 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:28:59 +0800
Subject: [PATCH 0910/1515] [youku] Better handling for Python 2/3
 compatibility

---
 youtube_dl/extractor/youku.py | 37 ++++++++++++-----------------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 4e47fca8a..26e5baadc 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -7,9 +7,10 @@ import base64
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
-from ..compat import compat_urllib_parse
-
-bytes_is_str = (bytes == str)  # for compatible
+from ..compat import (
+    compat_urllib_parse,
+    compat_ord,
+)
 
 
 class YoukuIE(InfoExtractor):
@@ -37,26 +38,20 @@ class YoukuIE(InfoExtractor):
             ls = list(range(256))
             t = 0
             for i in range(256):
-                t = (t + ls[i] + ord(s1[i % len(s1)])) % 256
+                t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s = '' if not bytes_is_str else b''
+            s = bytearray()
             x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
-                if isinstance(s2[i], int):
-                    s += chr(s2[i] ^ ls[(ls[x] + ls[y]) % 256])
-                else:
-                    s += chr(ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
-            return s
+                s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
+            return bytes(s)
 
         sid, token = yk_t(
-            'becaf9be',
-            base64.b64decode(bytes(data2['ep'], 'ascii'))
-            if not bytes_is_str
-            else base64.b64decode(data2['ep'])
-        ).split('_')
+            b'becaf9be', base64.b64decode(data2['ep'].encode('ascii'))
+        ).decode('ascii').split('_')
 
         # get oip
         oip = data2['ip']
@@ -89,16 +84,10 @@ class YoukuIE(InfoExtractor):
         def generate_ep(format, n):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
-                'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii')
-                if not bytes_is_str
-                else ('%s_%s_%s' % (sid, fileid, token))
+                b'bf7e5f01',
+                ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
             )
-            ep = base64.b64encode(
-                bytes(ep_t, 'latin')
-                if not bytes_is_str
-                else ep_t
-            ).decode()
+            ep = base64.b64encode(ep_t).decode('ascii')
             return ep
 
         # generate video_urls

From 99e6833c85868b78df7c810603ffdccdaeb4eaf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:30:27 +0600
Subject: [PATCH 0911/1515] [francetv] Update f4m manifest token URL (Closes
 #5981, Closes #5989)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index edf555b29..db0bbec1e 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -60,7 +60,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     continue
                 video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
-                    'http://hdfauth.francetv.fr/esi/urltokengen2.html?url=%s' % video_url_parsed.path,
+                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url_parsed.path,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))

From 9383e66f9475eca0e64c09972c1392d92d17570c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:31:30 +0800
Subject: [PATCH 0912/1515] [youku] Use _match_id

---
 youtube_dl/extractor/youku.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 26e5baadc..e41b48369 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
@@ -154,8 +153,7 @@ class YoukuIE(InfoExtractor):
         return _dict[fm]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # request basic data
         data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id

From ee69799262e8344742b9d8b492fe792b4d586f6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:36:28 +0800
Subject: [PATCH 0913/1515] [youku] Add a v.swf test case

---
 youtube_dl/extractor/youku.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e41b48369..d8162a0c5 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -21,7 +21,7 @@ class YoukuIE(InfoExtractor):
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
@@ -29,7 +29,10 @@ class YoukuIE(InfoExtractor):
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'flv'
         }
-    }
+    }, {
+        'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
+        'only_matching': True,
+    }]
 
     def construct_video_urls(self, data1, data2):
         # get sid, token

From 4fd35ee072a39654f11e794db5c50ee375c9a7c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:36:30 +0600
Subject: [PATCH 0914/1515] [safari] Add test for #5985

---
 youtube_dl/extractor/safari.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 20ba6fa33..f3c80708c 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -100,6 +100,10 @@ class SafariIE(SafariBaseIE):
     }, {
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
         'only_matching': True,
+    }, {
+        # non-digits in course id
+        'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f1e66cb2eb40b48c6508acbe57207a2d99792bf0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:46:07 +0800
Subject: [PATCH 0915/1515] [youku] Change video_id and add a multipart test
 case

---
 youtube_dl/extractor/youku.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d8162a0c5..d5b73ebce 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -25,13 +25,20 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
-            'id': 'XMTc1ODE5Njcy',
+            'id': 'XMTc1ODE5Njcy_part1',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'flv'
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
         'only_matching': True,
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
+        'info_dict': {
+            'id': 'XODgxNjg1Mzk2',
+            'title': '武媚娘传奇 85',
+        },
+        'playlist_count': 11,
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -201,20 +208,14 @@ class YoukuIE(InfoExtractor):
         for i in range(len(entries)):
             entries[i].update(
                 {
-                    'id': '_part%d' % (i + 1),
+                    'id': '%s_part%d' % (video_id, i + 1),
                     'title': title,
                 }
             )
 
-        if len(entries) > 1:
-            info = {
-                '_type': 'multi_video',
-                'id': video_id,
-                'title': title,
-                'entries': entries,
-            }
-        else:
-            info = entries[0]
-            info['id'] = video_id
-
-        return info
+        return {
+            '_type': 'multi_video',
+            'id': video_id,
+            'title': title,
+            'entries': entries,
+        }

From 04e7596680bce28beae2436bac0f6d1f01a45210 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:54:55 +0800
Subject: [PATCH 0916/1515] [youku] Better error handling

---
 youtube_dl/extractor/youku.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d5b73ebce..91f9f6bff 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -176,13 +176,15 @@ class YoukuIE(InfoExtractor):
 
         error_code = data1.get('error_code')
         if error_code:
-            # -8 means blocked outside China.
-            # Chinese and English, separated by newline.
             error = data1.get('error')
-            raise ExtractorError(
-                error or 'Server reported error %i' %
-                error_code,
-                expected=True)
+            if error is not None and '因版权原因无法观看此视频' in error:
+                raise ExtractorError(
+                    'Youku said: Sorry, this video is available in China only', expected=True)
+            else:
+                msg = 'Youku server reported error %i' % error_code
+                if error is not None:
+                    msg += ': ' + error
+                raise ExtractorError(msg)
 
         title = data1['title']
 

From 5228b756af2c2bfc2962a5b1bb6db1e6a41c9e05 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:06:23 +0800
Subject: [PATCH 0917/1515] [youku] Add cn_verification_proxy support and add a
 georestricted test case

---
 youtube_dl/extractor/youku.py | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 91f9f6bff..ea37dc8b2 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -9,6 +9,7 @@ from ..utils import ExtractorError
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
+    compat_urllib_request,
 )
 
 
@@ -39,6 +40,14 @@ class YoukuIE(InfoExtractor):
             'title': '武媚娘传奇 85',
         },
         'playlist_count': 11,
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
+        'info_dict': {
+            'id': 'XMTI1OTczNDM5Mg',
+            'title': '花千骨 04',
+        },
+        'playlist_count': 13,
+        'skip': 'Available in China only',
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -165,14 +174,23 @@ class YoukuIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # request basic data
-        data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
-        data2_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id
+        def retrieve_data(req_url, note):
+            req = compat_urllib_request.Request(req_url)
 
-        raw_data1 = self._download_json(data1_url, video_id)
-        raw_data2 = self._download_json(data2_url, video_id)
-        data1 = raw_data1['data'][0]
-        data2 = raw_data2['data'][0]
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
+            raw_data = self._download_json(req, video_id, note=note)
+            return raw_data['data'][0]
+
+        # request basic data
+        data1 = retrieve_data(
+            'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
+            'Downloading JSON metadata 1')
+        data2 = retrieve_data(
+            'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            'Downloading JSON metadata 2')
 
         error_code = data1.get('error_code')
         if error_code:

From a155b7e76c5a71c650f62c4716d23a24943fc373 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:15:09 +0800
Subject: [PATCH 0918/1515] [youku] Coding style

---
 youtube_dl/extractor/youku.py | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ea37dc8b2..cab5be3a4 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -216,22 +216,18 @@ class YoukuIE(InfoExtractor):
             for i in range(len(video_urls)):
                 if len(entries) < i + 1:
                     entries.append({'formats': []})
-                entries[i]['formats'].append(
-                    {
-                        'url': video_urls[i],
-                        'format_id': self.get_format_name(fm),
-                        'ext': self.parse_ext_l(fm),
-                        'filesize': int(data1['segs'][fm][i]['size'])
-                    }
-                )
+                entries[i]['formats'].append({
+                    'url': video_urls[i],
+                    'format_id': self.get_format_name(fm),
+                    'ext': self.parse_ext_l(fm),
+                    'filesize': int(data1['segs'][fm][i]['size'])
+                })
 
         for i in range(len(entries)):
-            entries[i].update(
-                {
-                    'id': '%s_part%d' % (video_id, i + 1),
-                    'title': title,
-                }
-            )
+            entries[i].update({
+                'id': '%s_part%d' % (video_id, i + 1),
+                'title': title,
+            })
 
         return {
             '_type': 'multi_video',

From 0501bfa159db5b5e8ed7fd1ed966b9989becb3e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:15:30 +0800
Subject: [PATCH 0919/1515] [YoutubeDL] Youku extractor now uses the standard
 format selection

---
 youtube_dl/YoutubeDL.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aacec2958..6e4b6f566 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1033,12 +1033,6 @@ class YoutubeDL(object):
             info_dict['id'], info_dict.get('subtitles'),
             info_dict.get('automatic_captions'))
 
-        # This extractors handle format selection themselves
-        if info_dict['extractor'] in ['Youku']:
-            if download:
-                self.process_info(info_dict)
-            return info_dict
-
         # We now pick which formats have to be downloaded
         if info_dict.get('formats') is None:
             # There's only one format available

From 7f0172b3e5e0da2a19708fdf3ec1b521a6e2656f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 22:29:41 +0600
Subject: [PATCH 0920/1515] Credit @jackyzy823 for iqiyi

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bf2a25cb8..889d599a2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -127,3 +127,4 @@ Julian Richen
 Ping O.
 Mister Hat
 Peter Ding
+jackyzy823

From f3aecb27a4d7b178ae66b4a294cff5dbe9bb2b18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Jun 2015 14:41:52 +0200
Subject: [PATCH 0921/1515] [youku] Simplify a bit the 'entries' construction

Mainly avoid having to use an index.
---
 youtube_dl/extractor/youku.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index cab5be3a4..ced3a10cd 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -210,25 +210,23 @@ class YoukuIE(InfoExtractor):
         video_urls_dict = self.construct_video_urls(data1, data2)
 
         # construct info
-        entries = []
+        entries = [{
+            'id': '%s_part%d' % (video_id, i + 1),
+            'title': title,
+            'formats': [],
+            # some formats are not available for all parts, we have to detect
+            # which one has all
+        } for i in range(max(len(v) for v in data1['segs'].values()))]
         for fm in data1['streamtypes']:
             video_urls = video_urls_dict[fm]
-            for i in range(len(video_urls)):
-                if len(entries) < i + 1:
-                    entries.append({'formats': []})
-                entries[i]['formats'].append({
-                    'url': video_urls[i],
+            for video_url, seg, entry in zip(video_urls, data1['segs'][fm], entries):
+                entry['formats'].append({
+                    'url': video_url,
                     'format_id': self.get_format_name(fm),
                     'ext': self.parse_ext_l(fm),
-                    'filesize': int(data1['segs'][fm][i]['size'])
+                    'filesize': int(seg['size']),
                 })
 
-        for i in range(len(entries)):
-            entries[i].update({
-                'id': '%s_part%d' % (video_id, i + 1),
-                'title': title,
-            })
-
         return {
             '_type': 'multi_video',
             'id': video_id,

From 447053668fbed993f6f4fd2e06d9282ea30224bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Jun 2015 21:19:18 +0600
Subject: [PATCH 0922/1515] [spankwire] Fix extraction

---
 youtube_dl/extractor/spankwire.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 06d6e6640..bff75d6b2 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -27,7 +27,7 @@ class SpankwireIE(InfoExtractor):
             'description': 'Crazy Bitch X rated music video.',
             'uploader': 'oreusz',
             'uploader_id': '124697',
-            'upload_date': '20070508',
+            'upload_date': '20070507',
             'age_limit': 18,
         }
     }
@@ -44,7 +44,7 @@ class SpankwireIE(InfoExtractor):
         title = self._html_search_regex(
             r'<h1>([^<]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'<div\s+id="descriptionContent">([^<]+)<',
+            r'(?s)<div\s+id="descriptionContent">(.+?)</div>',
             webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
             r'playerData\.screenShot\s*=\s*["\']([^"\']+)["\']',
@@ -64,12 +64,12 @@ class SpankwireIE(InfoExtractor):
             r'<div id="viewsCounter"><span>([\d,\.]+)</span> views</div>',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'Comments<span[^>]+>\s*\(([\d,\.]+)\)</span>',
+            r'<span\s+id="spCommentCount"[^>]*>([\d,\.]+)</span>',
             webpage, 'comment count', fatal=False))
 
         video_urls = list(map(
             compat_urllib_parse.unquote,
-            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
+            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',

From 14835de9fb41798c8e6e731a3f07ae871770666f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 16 Jun 2015 18:10:31 -0300
Subject: [PATCH 0923/1515] Use shlex.split for --pp-params and update related
 docs.

---
 README.md                          | 2 +-
 youtube_dl/YoutubeDL.py            | 1 +
 youtube_dl/__init__.py             | 6 ++++--
 youtube_dl/options.py              | 4 ++--
 youtube_dl/postprocessor/common.py | 3 ++-
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 726ec9cf2..813ac4a15 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --pp-params                      Extra parameters for video post-processor. The params will be splited on spaces.
+    --pp-params                      Extra parameters for video post-processor.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..3bfe30c76 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -261,6 +261,7 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
+    pp_params:         Extra parameters for external apps, like avconv.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5b28e4817..8b54d4ae2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -171,8 +171,10 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
-    if opts.pp_params is not None:
-        opts.pp_params = opts.pp_params.split()
+    if opts.pp_params is None:
+        opts.pp_params = []
+    else:
+        opts.pp_params = shlex.split(opts.pp_params)
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ceb4b5f38..fbba9b9d8 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -689,8 +689,8 @@ def parseOpts(overrideArguments=None):
         help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
     postproc.add_option(
         '--pp-params',
-        dest='pp_params', default=None,
-        help='Extra parameters for video post-processor. The params will be splited on spaces.')
+        dest='pp_params', default=None, metavar='ARGS',
+        help='Extra parameters for video post-processor.')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 3b0e8ddd8..d944d9367 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -22,7 +22,8 @@ class PostProcessor(object):
     of the chain is reached.
 
     PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects.
+    to InfoExtractor objects. And it can receive parameters from CLI trough
+    --pp-params.
     """
 
     _downloader = None

From 028a33d7f2a0bc028f533530d2722b57b31dabdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:27:38 +0600
Subject: [PATCH 0924/1515] [lifenews] Fix extraction

---
 youtube_dl/extractor/lifenews.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 42cb6e35f..373122c93 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -82,10 +82,11 @@ class LifeNewsIE(InfoExtractor):
         view_count = self._html_search_regex(
             r'<div class=\'views\'>\s*(\d+)\s*</div>', webpage, 'view count', fatal=False)
         comment_count = self._html_search_regex(
-            r'<div class=\'comments\'>\s*<span class=\'counter\'>\s*(\d+)\s*</span>', webpage, 'comment count', fatal=False)
+            r'=\'commentCount\'[^>]*>\s*(\d+)\s*<',
+            webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
-            r'<time datetime=\'([^\']+)\'>', webpage, 'upload date', fatal=False)
+            r'<time[^>]*datetime=\'([^\']+)\'', webpage, 'upload date', fatal=False)
         if upload_date is not None:
             upload_date = unified_strdate(upload_date)
 

From 062a3fdf36dee6515c9e3b103c39ac2691196e95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:28:47 +0600
Subject: [PATCH 0925/1515] [lifenews] Modernize

---
 youtube_dl/extractor/lifenews.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 373122c93..1f07db3ec 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     int_or_none,
+    remove_end,
     unified_strdate,
     ExtractorError,
 )
@@ -72,10 +73,9 @@ class LifeNewsIE(InfoExtractor):
         if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 
-        title = self._og_search_title(webpage)
-        TITLE_SUFFIX = ' - Первый по срочным новостям — LIFE | NEWS'
-        if title.endswith(TITLE_SUFFIX):
-            title = title[:-len(TITLE_SUFFIX)]
+        title = remove_end(
+            self._og_search_title(webpage),
+            ' - Первый по срочным новостям — LIFE | NEWS')
 
         description = self._og_search_description(webpage)
 

From a69e8bfdd9e94537eb72a358c1b62dbea8a59a04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:29:15 +0600
Subject: [PATCH 0926/1515] [lifenews] Fix tests

---
 youtube_dl/extractor/lifenews.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 1f07db3ec..f8cbca7b3 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -40,7 +40,6 @@ class LifeNewsIE(InfoExtractor):
             'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
             'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
             'upload_date': '20150402',
-            'uploader': 'embed.life.ru',
         }
     }, {
         'url': 'http://lifenews.ru/news/153461',
@@ -51,7 +50,6 @@ class LifeNewsIE(InfoExtractor):
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
             'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
             'upload_date': '20150505',
-            'uploader': 'embed.life.ru',
         }
     }, {
         'url': 'http://lifenews.ru/video/13035',

From 0d0d5d37174ed611dd823c5be025c49e73d83d1d Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 13:59:37 +0800
Subject: [PATCH 0927/1515] [qqmusic] Add support for playlists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..d03577cdf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -427,6 +427,7 @@ from .qqmusic import (
     QQMusicSingerIE,
     QQMusicAlbumIE,
     QQMusicToplistIE,
+    QQMusicPlaylistIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..f9aafcd28 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    clean_html,
 )
 from ..compat import compat_urllib_request
 
@@ -243,3 +244,36 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_name = topinfo.get('ListName')
         list_description = topinfo.get('info')
         return self.playlist_result(entries, list_id, list_name, list_description)
+
+
+class QQMusicPlaylistIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:playlist'
+    _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://y.qq.com/#type=taoge&id=3462654915',
+        'info_dict': {
+            'id': '3462654915',
+            'title': '韩国5月新歌精选下旬',
+            'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
+        },
+        'playlist_count': 40,
+    }
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        list_json = self._download_json(
+            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
+            % list_id, list_id, 'Download list page',
+            transform_source=strip_jsonp)['cdlist'][0]
+
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+            ) for song in list_json['songlist']
+        ]
+
+        list_name = list_json['dissname']
+        list_description = clean_html(unescapeHTML(list_json.get('desc')))
+        return self.playlist_result(entries, list_id, list_name, list_description)

From 8f73e89ca0ecde0a8bbd1f1463e9a06a53c6b573 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 14:15:54 +0800
Subject: [PATCH 0928/1515] [kuwo] New extractor for kuwo.cn

---
 youtube_dl/extractor/__init__.py |   9 +
 youtube_dl/extractor/kuwo.py     | 326 +++++++++++++++++++++++++++++++
 2 files changed, 335 insertions(+)
 create mode 100644 youtube_dl/extractor/kuwo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..82cd85c44 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -257,6 +257,15 @@ from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
+from .kuwo import (
+    KuwoIE,
+    KuwoAlbumIE,
+    KuwoChartIE,
+    KuwoSingerIE,
+    KuwoSingerMusicIE,
+    KuwoCategoryIE,
+    KuwoMvIE,
+)
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .letv import (
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
new file mode 100644
index 000000000..6a96a1aa4
--- /dev/null
+++ b/youtube_dl/extractor/kuwo.py
@@ -0,0 +1,326 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    get_element_by_id,
+    clean_html,
+    ExtractorError,
+)
+
+
+class KuwoIE(InfoExtractor):
+    IE_NAME = 'kuwo:song'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/yinyue/635632/',
+        'info_dict': {
+            'id': '635632',
+            'ext': 'ape',
+            'title': '爱我别走',
+            'creator': '张震岳',
+            'upload_date': '20080122',
+            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
+        },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/6446136/',
+        'info_dict': {
+            'id': '6446136',
+            'ext': 'mp3',
+            'title': '心',
+            'creator': 'IU',
+            'upload_date': '20150518',
+        },
+        'params': {
+            'format': 'mp3-320'
+        },
+    }]
+    _FORMATS = [
+        {'format': 'ape', 'ext': 'ape', 'preference': 100},
+        {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
+        {'format': 'mp3-192', 'ext': 'mp3', 'br': '192kmp3', 'abr': 192, 'preference': 70},
+        {'format': 'mp3-128', 'ext': 'mp3', 'br': '128kmp3', 'abr': 128, 'preference': 60},
+        {'format': 'wma', 'ext': 'wma', 'preference': 20},
+        {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
+    ]
+
+    def _get_formats(self, song_id):
+        formats = []
+        for file_format in self._FORMATS:
+            song_url = self._download_webpage(
+                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
+                (file_format['ext'], file_format.get('br', ''), song_id),
+                song_id, note="Download %s url info" % file_format["format"],
+            )
+            if song_url.startswith('http://') or song_url.startswith('https://'):
+                formats.append({
+                    'url': song_url,
+                    'format_id': file_format['format'],
+                    'format': file_format['format'],
+                    'preference': file_format['preference'],
+                    'abr': file_format.get('abr'),
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, song_id, note='Download song detail info',
+            errnote='Unable to get song detail info')
+
+        song_name = self._html_search_regex(
+            r'<h1 title="(.+?)">', webpage, 'song name')
+        singer_name = self._html_search_regex(
+            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
+            flags=re.DOTALL, default=None)
+        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
+        if lrc_content == '暂无':     # indicates no lyrics
+            lrc_content = None
+
+        formats = self._get_formats(song_id)
+
+        album_id = self._html_search_regex(
+            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
+            webpage, 'album id', default=None, fatal=False)
+
+        publish_time = None
+        if album_id is not None:
+            album_info_page = self._download_webpage(
+                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
+                note='Download album detail info',
+                errnote='Unable to get album detail info')
+
+            publish_time = self._html_search_regex(
+                r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
+                'publish time', default=None)
+            if publish_time:
+                publish_time = publish_time.replace('-', '')
+
+        return {
+            'id': song_id,
+            'title': song_name,
+            'creator': singer_name,
+            'upload_date': publish_time,
+            'description': lrc_content,
+            'formats': formats,
+        }
+
+
+class KuwoAlbumIE(InfoExtractor):
+    IE_NAME = 'kuwo:album'
+    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/album/502294/',
+        'info_dict': {
+            'id': '502294',
+            'title': 'M',
+            'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
+        },
+        'playlist_count': 2,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, album_id, note='Download album info',
+            errnote='Unable to get album info')
+
+        album_name = self._html_search_regex(
+            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
+            'album name', flags=re.DOTALL)
+        album_intro = clean_html(
+            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
+                webpage)
+        ]
+        return self.playlist_result(entries, album_id, album_name, album_intro)
+
+
+class KuwoChartIE(InfoExtractor):
+    IE_NAME = 'kuwo:chart'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
+    _TEST = {
+        'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
+        'info_dict': {
+            'id': '香港中文龙虎榜',
+            'title': '香港中文龙虎榜',
+            'description': 're:[0-9]{4}第[0-9]{2}期',
+        },
+        'playlist_mincount': 10,
+    }
+
+    def _real_extract(self, url):
+        chart_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, chart_id, note='Download chart info',
+            errnote='Unable to get chart info')
+
+        chart_name = self._html_search_regex(
+            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
+
+        chart_desc = self._html_search_regex(
+            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
+        ]
+        return self.playlist_result(entries, chart_id, chart_name, chart_desc)
+
+
+class KuwoSingerIE(InfoExtractor):
+    IE_NAME = 'kuwo:singer'
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/$'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
+        'info_dict': {
+            'id': 'bruno+mars',
+            'title': 'Bruno Mars',
+        },
+        'playlist_count': 10,
+    }
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, singer_id, note='Download singer info',
+            errnote='Unable to get singer info')
+
+        singer_name = self._html_search_regex(
+            r'姓名：<span>(.+?)</span>', webpage, 'singer name')
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>',
+                webpage, flags=re.DOTALL)
+        ]
+        return self.playlist_result(entries, singer_id, singer_name)
+
+
+class KuwoSingerMusicIE(InfoExtractor):
+    IE_NAME = 'kuwo:singermusic'
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/music(_[0-9]+)?.htm'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
+        'info_dict': {
+            'id': 'Ali',
+            'title': 'Ali的热门歌曲',
+        },
+        'playlist_mincount': 95,
+    }
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+
+        list_name = None
+        entries = []
+        for page_num in itertools.count(1):
+            webpage = self._download_webpage(
+                'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
+                singer_id, note='Download song list page #%d' % page_num,
+                errnote='Unable to get song list page #%d' % page_num)
+
+            if list_name is None:
+                list_name = self._html_search_regex(
+                    r'<h1>([^<>]+)<span>', webpage, 'list name')
+
+            entries.extend([
+                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+                for song_id in re.findall(
+                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                    webpage)
+            ])
+            if not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+                break
+
+        return self.playlist_result(entries, singer_id, list_name)
+
+
+class KuwoCategoryIE(InfoExtractor):
+    IE_NAME = 'kuwo:category'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
+    _TEST = {
+        'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
+        'info_dict': {
+            'id': '86375',
+            'title': '八十年代精选',
+            'description': '这些都是属于八十年代的回忆！',
+        },
+        'playlist_count': 30,
+    }
+
+    def _real_extract(self, url):
+        category_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, category_id, note='Download category info',
+            errnote='Unable to get category info')
+
+        category_name = self._html_search_regex(
+            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
+        
+        category_desc = re.sub(
+            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
+        
+        jsonm = self._parse_json(self._html_search_regex(
+            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
+
+        entries = [
+            self.url_result(
+                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
+                'Kuwo', song['musicrid'])
+            for song in jsonm['musiclist']
+        ]
+        return self.playlist_result(entries, category_id, category_name, category_desc)
+
+
+class KuwoMvIE(KuwoIE):
+    IE_NAME = 'kuwo:mv'
+    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/mv/6480076/',
+        'info_dict': {
+            'id': '6480076',
+            'ext': 'mkv',
+            'title': '我们家MV',
+            'creator': '2PM',
+        },
+    }]
+    _FORMATS = KuwoIE._FORMATS + [
+        {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
+        {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
+    ]
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, song_id, note='Download mv detail info: %s' % song_id,
+            errnote='Unable to get mv detail info: %s' % song_id)
+
+        mobj = re.search(
+            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
+            webpage)
+        if mobj:
+            song_name = mobj.group('song')
+            singer_name = mobj.group('singer')
+        else:
+            raise ExtractorError("Unable to find song or singer names")
+
+        formats = self._get_formats(song_id)
+
+        return {
+            'id': song_id,
+            'title': song_name,
+            'creator': singer_name,
+            'formats': formats,
+        }

From 29b809de68aeefb5e991c75929ed3d03fb40c1f1 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 15:52:04 +0800
Subject: [PATCH 0929/1515] [qqmusic] Fix album extraction

---
 youtube_dl/extractor/qqmusic.py | 37 ++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..d9a783f8a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -163,31 +163,38 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
-    _TEST = {
-        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1&play=0',
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
-            'description': 'md5:d216c55a2d4b3537fe4415b8767d74d6',
+            'description': 'md5:712f0cdbfc7e776820d08150e6df593d',
         },
         'playlist_count': 4,
-    }
+    }, {
+        'url': 'http://y.qq.com/#type=album&mid=002Y5a3b3AlCu3',
+        'info_dict': {
+            'id': '002Y5a3b3AlCu3',
+            'title': '그리고...',
+            'description': 'md5:b1d133b8c9bac8fed4e1a97df759f4cf',
+        },
+        'playlist_count': 8,
+    }]
 
     def _real_extract(self, url):
         mid = self._match_id(url)
 
-        album_page = self._download_webpage(
-            self.qq_static_url('album', mid), mid, 'Download album page')
+        album = self._download_json(
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_album_info_cp.fcg?albummid=%s&format=json' % mid,
+            mid, 'Download album page')['data']
 
-        entries = self.get_entries_from_page(album_page)
-
-        album_name = self._html_search_regex(
-            r"albumname\s*:\s*'([^']+)',", album_page, 'album name',
-            default=None)
-
-        album_detail = self._html_search_regex(
-            r'<div class="album_detail close_detail">\s*<p>((?:[^<>]+(?:<br />)?)+)</p>',
-            album_page, 'album details', default=None)
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+            ) for song in album['list']
+        ]
+        album_name = album['name']
+        album_detail = album.get('desc')
 
         return self.playlist_result(entries, mid, album_name, album_detail)
 

From 5e3915cbe3fabd2dbb633131b851b1158c0bba7b Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 21:06:25 +0800
Subject: [PATCH 0930/1515] [qqmusic] Fix song extraction when certain formats
 are unavailable

---
 youtube_dl/extractor/qqmusic.py | 55 +++++++++++++++++++++++++++------
 1 file changed, 46 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..7ddc4ca25 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,8 +9,13 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    HEADRequest,
+    ExtractorError,
+)
+from ..compat import (
+    compat_urllib_request,
+    compat_HTTPError,
 )
-from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -26,6 +31,20 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
+            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/7/p/001IV22P1RDX7p.jpg',
+        }
+    }, {
+        'note': 'There is no mp3-320 version of this song.',
+        'url': 'http://y.qq.com/#type=song&mid=004MsGEo3DdNxV',
+        'md5': 'fa3926f0c585cda0af8fa4f796482e3e',
+        'info_dict': {
+            'id': '004MsGEo3DdNxV',
+            'ext': 'mp3',
+            'title': '如果',
+            'upload_date': '20050626',
+            'creator': '李季美',
+            'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
+            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/r/Q/0042owYj46IxrQ.jpg',
         }
     }]
 
@@ -68,6 +87,13 @@ class QQMusicIE(InfoExtractor):
         if lrc_content:
             lrc_content = lrc_content.replace('\\n', '\n')
 
+        thumbnail_url = None
+        albummid = self._search_regex(
+            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'], detail_info_page, 'album mid', default=None)
+        if albummid:
+            thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
+                            % (albummid[-2:-1], albummid[-1], albummid)
+
         guid = self.m_r_get_ruin()
 
         vkey = self._download_json(
@@ -77,14 +103,24 @@ class QQMusicIE(InfoExtractor):
 
         formats = []
         for format_id, details in self._FORMATS.items():
-            formats.append({
-                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (details['prefix'], mid, details['ext'], vkey, guid),
-                'format': format_id,
-                'format_id': format_id,
-                'preference': details['preference'],
-                'abr': details.get('abr'),
-            })
+            video_url = 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0' \
+                        % (details['prefix'], mid, details['ext'], vkey, guid)
+            req = HEADRequest(video_url)
+            try:
+                res = self._request_webpage(
+                    req, mid, note='Testing %s video URL' % format_id, fatal=False)
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [400, 404]:
+                    self.report_warning('Invalid %s video URL' % format_id, mid)
+            else:
+                if res:
+                    formats.append({
+                        'url': video_url,
+                        'format': format_id,
+                        'format_id': format_id,
+                        'preference': details['preference'],
+                        'abr': details.get('abr'),
+                    })
         self._sort_formats(formats)
 
         return {
@@ -94,6 +130,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': publish_time,
             'creator': singer,
             'description': lrc_content,
+            'thumbnail': thumbnail_url,
         }
 
 

From 0392ac98d2c5c5a6fd2ab51c51096f82312a287c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 21:13:03 +0800
Subject: [PATCH 0931/1515] [qqmusic] Fix code formatting

---
 youtube_dl/extractor/qqmusic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 7ddc4ca25..5a18191bc 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -89,7 +89,8 @@ class QQMusicIE(InfoExtractor):
 
         thumbnail_url = None
         albummid = self._search_regex(
-            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'], detail_info_page, 'album mid', default=None)
+            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'],
+            detail_info_page, 'album mid', default=None)
         if albummid:
             thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
                             % (albummid[-2:-1], albummid[-1], albummid)

From 974a6146fe71029a8dedf670fd72501da5ec3809 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 21:40:45 +0800
Subject: [PATCH 0932/1515] [bbccouk] Fix error handling

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 249bc6bbd..4b25b8c38 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -267,7 +267,7 @@ class BBCCoUkIE(InfoExtractor):
                 programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
 

From 25fa8d66e653d443522bf10140106f88711d1d66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 22:00:13 +0800
Subject: [PATCH 0933/1515] [bbccouk] Fix description (closes #6006)

---
 youtube_dl/extractor/bbccouk.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 4b25b8c38..0305f88b5 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -129,6 +129,20 @@ class BBCCoUkIE(InfoExtractor):
                 'skip_download': True,
             },
             'skip': 'geolocation',
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
+            'info_dict': {
+                'id': 'b05zmgw1',
+                'ext': 'flv',
+                'description': 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.',
+                'title': 'Royal Academy Summer Exhibition',
+                'duration': 3540,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'geolocation',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,
@@ -362,7 +376,7 @@ class BBCCoUkIE(InfoExtractor):
             formats, subtitles = self._download_media_selector(programme_id)
             title = self._og_search_title(webpage)
             description = self._search_regex(
-                r'<p class="medium-description">([^<]+)</p>',
+                r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', fatal=False)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)

From c33a8639a7c1f783297f29282b48fceec9a0b84a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 22:05:02 +0800
Subject: [PATCH 0934/1515] [postprocessor/embedthumbnail] Skipping if
 thumbnail file missing

Fixes #5996
---
 youtube_dl/postprocessor/embedthumbnail.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 774494efd..e19dbf73d 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -35,6 +35,11 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
 
         thumbnail_filename = info['thumbnails'][-1]['filename']
 
+        if not os.path.exists(encodeFilename(thumbnail_filename)):
+            self._downloader.report_warning(
+                'Skipping embedding the thumbnail because the file is missing.')
+            return [], info
+
         if info['ext'] == 'mp3':
             options = [
                 '-c', 'copy', '-map', '0', '-map', '1',

From 4d58b24c15ea0efc699a7ad7ee468245029da4e3 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 23:09:04 +0800
Subject: [PATCH 0935/1515] [qqmusic] Use _check_formats instead

---
 youtube_dl/extractor/qqmusic.py | 34 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 5a18191bc..7183c2bb1 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,13 +9,8 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
-    HEADRequest,
-    ExtractorError,
-)
-from ..compat import (
-    compat_urllib_request,
-    compat_HTTPError,
 )
+from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -104,24 +99,15 @@ class QQMusicIE(InfoExtractor):
 
         formats = []
         for format_id, details in self._FORMATS.items():
-            video_url = 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0' \
-                        % (details['prefix'], mid, details['ext'], vkey, guid)
-            req = HEADRequest(video_url)
-            try:
-                res = self._request_webpage(
-                    req, mid, note='Testing %s video URL' % format_id, fatal=False)
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [400, 404]:
-                    self.report_warning('Invalid %s video URL' % format_id, mid)
-            else:
-                if res:
-                    formats.append({
-                        'url': video_url,
-                        'format': format_id,
-                        'format_id': format_id,
-                        'preference': details['preference'],
-                        'abr': details.get('abr'),
-                    })
+            formats.append({
+                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
+                       % (details['prefix'], mid, details['ext'], vkey, guid),
+                'format': format_id,
+                'format_id': format_id,
+                'preference': details['preference'],
+                'abr': details.get('abr'),
+            })
+        self._check_formats(formats, mid)
         self._sort_formats(formats)
 
         return {

From accf79b1079302a339a3565e98ecfea5b33b5528 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Jun 2015 21:17:26 +0600
Subject: [PATCH 0936/1515] [viki] Add support for authentication (Closes
 #6005)

---
 youtube_dl/extractor/viki.py | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 7f2fb1ca8..52d10d242 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,5 +1,7 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import time
 import hmac
 import hashlib
@@ -11,6 +13,7 @@ from ..utils import (
     parse_age_limit,
     parse_iso8601,
 )
+from ..compat import compat_urllib_request
 from .common import InfoExtractor
 
 
@@ -23,7 +26,9 @@ class VikiBaseIE(InfoExtractor):
     _APP_VERSION = '2.2.5.1428709186'
     _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
 
-    def _prepare_call(self, path, timestamp=None):
+    _NETRC_MACHINE = 'viki'
+
+    def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
             timestamp = int(time.time())
@@ -33,17 +38,19 @@ class VikiBaseIE(InfoExtractor):
             query.encode('ascii'),
             hashlib.sha1
         ).hexdigest()
-        return self._API_URL_TEMPLATE % (query, sig)
+        url = self._API_URL_TEMPLATE % (query, sig)
+        return compat_urllib_request.Request(
+            url, json.dumps(post_data).encode('utf-8')) if post_data else url
 
-    def _call_api(self, path, video_id, note, timestamp=None):
+    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
         resp = self._download_json(
-            self._prepare_call(path, timestamp), video_id, note)
+            self._prepare_call(path, timestamp, post_data), video_id, note)
 
         error = resp.get('error')
         if error:
             if error == 'invalid timestamp':
                 resp = self._download_json(
-                    self._prepare_call(path, int(resp['current_timestamp'])),
+                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
                     video_id, '%s (retry)' % note)
                 error = resp.get('error')
             if error:
@@ -56,6 +63,23 @@ class VikiBaseIE(InfoExtractor):
             '%s returned error: %s' % (self.IE_NAME, error),
             expected=True)
 
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'login_id': username,
+            'password': password,
+        }
+
+        self._call_api(
+            'sessions.json', None,
+            'Logging in as %s' % username, post_data=login_form)
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'

From 360075e28a4c009babb62e0ed8141ed5244cb3f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Jun 2015 22:26:17 +0600
Subject: [PATCH 0937/1515] [pornhub] Extend _VALID_URL (Closes #6019)

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 8565d7551..8172bc997 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -19,8 +19,8 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-f]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
+    _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
         'info_dict': {
@@ -30,7 +30,10 @@ class PornHubIE(InfoExtractor):
             "title": "Seductive Indian beauty strips down and fingers her pink pussy",
             "age_limit": 18
         }
-    }
+    }, {
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_url(cls, webpage):

From 9e96dc8b3561c1e6e62ce6a34efba485e5e49054 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 01:36:59 -0500
Subject: [PATCH 0938/1515] Support BBC News (bbc.com/news)

---
 docs/supportedsites.md           |   1 +
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/bbcnews.py  | 162 +++++++++++++++++++++++++++++++
 3 files changed, 164 insertions(+)
 create mode 100644 youtube_dl/extractor/bbcnews.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 220e52b98..d4ccbbd3a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,6 +50,7 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
+ - **bbc.com**: BBC news videos
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..51d2d20e9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -36,6 +36,7 @@ from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
+from .bbcnews import BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
new file mode 100644
index 000000000..b10e30a81
--- /dev/null
+++ b/youtube_dl/extractor/bbcnews.py
@@ -0,0 +1,162 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+from ..compat import compat_HTTPError
+import re
+from .bbccouk import BBCCoUkIE
+
+class BBCNewsIE(BBCCoUkIE):
+    IE_NAME = 'bbc.com'
+    IE_DESC = 'BBC news'
+    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+        },
+        'playlist_count': 2,
+    },{
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+        },
+        'playlist_count': 9,
+    },{
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'mp4',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _duration_str2int(self, str):
+        if not str:
+            return None
+        ret = re.match(r'^\d+$', str)
+        if ret:
+            return int(ret.group(0))
+        ret = re.match(r'PT((?P<h>\d+)H)?((?P<m>\d+)M)?(?P<s>\d+)S$', str)
+        if ret:
+            total=int(ret.group('s'))
+            if ret.group('m'):
+                total+=(int(ret.group('m'))*60)
+            if ret.group('h'):
+                total+=(int(ret.group('h'))*3600)
+            return total
+        return None
+
+    def _download_media_selector(self, programme_id):
+        # bbc news uses http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ not
+        # http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/
+        # Could add third urlspec arg to BBCCoUkIE._download_media_selector instead of duplicating it
+
+        try:
+            media_selection = self._download_xml(
+               'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' % programme_id,
+                programme_id, 'Downloading media selection XML')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+            else:
+                raise
+        formats = []
+        subtitles = None
+
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self.extract_subtitles(media, programme_id)
+
+        formats = []
+        subtitles = None
+
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self.extract_subtitles(media, programme_id)
+
+        return formats, subtitles
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+
+        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
+        if pubdate:
+           pubdate = pubdate.replace('-','')
+
+        ret = []
+        # works with bbc.com/news/something-something-123456 articles
+        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
+        if not matches:
+           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+           # in http://www.bbc.com/news/video_and_audio/international
+           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+        if not matches:
+           raise ExtractorError('No video found', expected=True)
+
+        for ent in matches:
+            jent = self._parse_json(ent,list_id)
+
+            programme_id = jent.get('externalId',None)
+            xml_url = jent.get('href', None)
+
+            title = jent['caption']
+            duration = self._duration_str2int(jent.get('duration',None))
+            description = list_title + ' - ' + jent.get('caption','')
+            thumbnail = None
+            if jent.has_key('image'):
+               thumbnail=jent['image'].get('href',None)
+
+            if programme_id:
+               formats, subtitles = self._download_media_selector(programme_id)
+            elif xml_url:
+               # Cheap fallback
+               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+               formats, subtitles = self._download_media_selector(programme_id)
+            else:
+               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+               
+            self._sort_formats(formats)
+
+            ret.append( {
+                'id': programme_id,
+                'uploader': 'BBC News',
+                'upload_date': pubdate,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            } )
+
+        if len(ret) > 0:
+           return self.playlist_result(ret, list_id, list_title)
+        raise ExtractorError('No video found', expected=True)

From a8b081a0523c412fd4e01d5cddec7ae382c4793e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 01:52:25 -0500
Subject: [PATCH 0939/1515] BBCNewsIE: eliminate redundant function.
 BBCCoUkIE._download_media_selector: use class variable instead of hardcoded
 string for mediaselector_url template.

---
 youtube_dl/extractor/bbccouk.py |  4 +++-
 youtube_dl/extractor/bbcnews.py | 42 ++-------------------------------
 2 files changed, 5 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 0305f88b5..dcc5fc2fa 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -15,6 +15,8 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -277,7 +279,7 @@ class BBCCoUkIE(InfoExtractor):
     def _download_media_selector(self, programme_id):
         try:
             media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
+                self.mediaselector_url % programme_id,
                 programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
index b10e30a81..9bb8d42e6 100644
--- a/youtube_dl/extractor/bbcnews.py
+++ b/youtube_dl/extractor/bbcnews.py
@@ -14,6 +14,8 @@ class BBCNewsIE(BBCCoUkIE):
     IE_DESC = 'BBC news'
     _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
 
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+
     _TESTS = [{
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
@@ -59,46 +61,6 @@ class BBCNewsIE(BBCCoUkIE):
             return total
         return None
 
-    def _download_media_selector(self, programme_id):
-        # bbc news uses http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ not
-        # http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/
-        # Could add third urlspec arg to BBCCoUkIE._download_media_selector instead of duplicating it
-
-        try:
-            media_selection = self._download_xml(
-               'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' % programme_id,
-                programme_id, 'Downloading media selection XML')
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
-            else:
-                raise
-        formats = []
-        subtitles = None
-
-        for media in self._extract_medias(media_selection):
-            kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
-            elif kind == 'captions':
-                subtitles = self.extract_subtitles(media, programme_id)
-
-        formats = []
-        subtitles = None
-
-        for media in self._extract_medias(media_selection):
-            kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
-            elif kind == 'captions':
-                subtitles = self.extract_subtitles(media, programme_id)
-
-        return formats, subtitles
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)

From d5552a3477a0970f4aaaa746ce07c816267bb9cf Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 06:25:50 -0500
Subject: [PATCH 0940/1515] bbcnews: Switch to parse_duration, revert change to
 docs/supportedsites.md

---
 docs/supportedsites.md          |  1 -
 youtube_dl/extractor/bbcnews.py | 19 ++-----------------
 2 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d4ccbbd3a..220e52b98 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,7 +50,6 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
- - **bbc.com**: BBC news videos
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
index 9bb8d42e6..fd4a5e38f 100644
--- a/youtube_dl/extractor/bbcnews.py
+++ b/youtube_dl/extractor/bbcnews.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    parse_duration,
     int_or_none,
 )
 from ..compat import compat_HTTPError
@@ -45,22 +46,6 @@ class BBCNewsIE(BBCCoUkIE):
         }
     }]
 
-    def _duration_str2int(self, str):
-        if not str:
-            return None
-        ret = re.match(r'^\d+$', str)
-        if ret:
-            return int(ret.group(0))
-        ret = re.match(r'PT((?P<h>\d+)H)?((?P<m>\d+)M)?(?P<s>\d+)S$', str)
-        if ret:
-            total=int(ret.group('s'))
-            if ret.group('m'):
-                total+=(int(ret.group('m'))*60)
-            if ret.group('h'):
-                total+=(int(ret.group('h'))*3600)
-            return total
-        return None
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
@@ -88,7 +73,7 @@ class BBCNewsIE(BBCCoUkIE):
             xml_url = jent.get('href', None)
 
             title = jent['caption']
-            duration = self._duration_str2int(jent.get('duration',None))
+            duration = parse_duration(jent.get('duration',None))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):

From c9bebed294dd29d9188265c8f7bfb0e1b43406ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 20:52:44 +0600
Subject: [PATCH 0941/1515] [youtube] Add itag 59 and 78 (Closes #5979)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9e2671192..a3da56c14 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -234,6 +234,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '44': {'ext': 'webm', 'width': 854, 'height': 480},
         '45': {'ext': 'webm', 'width': 1280, 'height': 720},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480},
 
 
         # 3d videos

From cbcd1a5474dd8b39e68b0d2bbc493701c655a2d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 21:57:31 +0600
Subject: [PATCH 0942/1515] [dramafever] Add support for authentication (Closes
 #6017)

---
 youtube_dl/extractor/dramafever.py | 38 ++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index a34aad486..cfbcddcef 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -6,6 +6,8 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
+    compat_urllib_parse,
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,7 +19,39 @@ from ..utils import (
 )
 
 
-class DramaFeverIE(InfoExtractor):
+class DramaFeverBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
+    _NETRC_MACHINE = 'dramafever'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'username': username,
+            'password': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if all(logout_pattern not in response
+               for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
+            error = self._html_search_regex(
+                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                response, 'error message', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+
+class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TEST = {
@@ -97,7 +131,7 @@ class DramaFeverIE(InfoExtractor):
         }
 
 
-class DramaFeverSeriesIE(InfoExtractor):
+class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{

From 10464af5d1d03a3461286a601ae7db91c5a8141c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 22:02:07 +0600
Subject: [PATCH 0943/1515] [dramafever:series] Fix extraction while
 authenticated

---
 youtube_dl/extractor/dramafever.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index cfbcddcef..ca41a3abf 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -185,8 +185,11 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
                 % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
                 series_id, 'Downloading episodes JSON page #%d' % page_num)
             for episode in episodes.get('value', []):
+                episode_url = episode.get('episode_url')
+                if not episode_url:
+                    continue
                 entries.append(self.url_result(
-                    compat_urlparse.urljoin(url, episode['episode_url']),
+                    compat_urlparse.urljoin(url, episode_url),
                     'DramaFever', episode.get('guid')))
             if page_num == episodes['num_pages']:
                 break

From 385c3e5e91680dcc6573f05e6b30fdf45048503e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Jun 2015 00:10:08 +0600
Subject: [PATCH 0944/1515] [pinkbike] Improve and simplify

---
 youtube_dl/extractor/pinkbike.py | 88 ++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 45c0b1377..745433b48 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -7,12 +7,14 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     remove_end,
-    remove_start
+    remove_start,
+    str_to_int,
+    unified_strdate,
 )
 
 
 class PinkbikeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pinkbike\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?pinkbike\.com/video/|es\.pinkbike\.org/i/kvid/kvid-y5\.swf\?id=)(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.pinkbike.com/video/402811/',
         'md5': '4814b8ca7651034cd87e3361d5c2155a',
@@ -20,67 +22,75 @@ class PinkbikeIE(InfoExtractor):
             'id': '402811',
             'ext': 'mp4',
             'title': 'Brandon Semenuk - RAW 100',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'location': 'Victoria, British Columbia, Canada',
-            'uploader_id': 'revelco',
-            'upload_date': '20150406',
             'description': 'Official release: www.redbull.ca/rupertwalker',
-            'duration': 100
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 100,
+            'upload_date': '20150406',
+            'uploader': 'revelco',
+            'location': 'Victoria, British Columbia, Canada',
+            'view_count': int,
+            'comment_count': int,
         }
     }, {
-        'url': 'http://www.pinkbike.com/video/406629/',
-        'md5': 'c7a3e19a2bd5cde5a1cda6b2b46caa74',
-        'info_dict': {
-            'id': '406629',
-            'ext': 'mp4',
-            'title': 'Chromag: Reece Wallace in Utah',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'location': 'Whistler, British Columbia, Canada',
-            'uploader_id': 'Chromagbikes',
-            'upload_date': '20150505',
-            'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
-            'duration': 180
-        }
+        'url': 'http://es.pinkbike.org/i/kvid/kvid-y5.swf?id=406629',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        title = remove_end(title, ' Video - Pinkbike')
+        webpage = self._download_webpage(
+            'http://www.pinkbike.com/video/%s' % video_id, video_id)
 
-        description = self._html_search_meta('description', webpage, 'description')
-        description = remove_start(description, title + '. ')
+        formats = []
+        for _, format_id, src in re.findall(
+            r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None))
+            formats.append({
+                'url': src,
+                'format_id': format_id,
+                'height': height,
+            })
+        self._sort_formats(formats)
 
+        title = remove_end(self._og_search_title(webpage), ' Video - Pinkbike')
+        description = self._html_search_regex(
+            r'(?s)id="media-description"[^>]*>(.+?)<',
+            webpage, 'description', default=None) or remove_start(
+            self._og_search_description(webpage), title + '. ')
+        thumbnail = self._og_search_thumbnail(webpage)
         duration = int_or_none(self._html_search_meta(
             'video:duration', webpage, 'duration'))
 
-        uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
-
-        upload_date = self._html_search_regex(
-            r'class="fullTime"\s*title="([0-9]{4}(?:-[0-9]{2}){2})"',
-            webpage, 'upload_date')
-        upload_date = upload_date.replace('-', '')
+        uploader = self._search_regex(
+            r'un:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
+        upload_date = unified_strdate(self._search_regex(
+            r'class="fullTime"[^>]+title="([^"]+)"',
+            webpage, 'upload date', fatal=False))
 
         location = self._html_search_regex(
-            r'<dt>Location</dt>\n?\s*<dd>\n?(.*?)\s*<img',
-            webpage, 'location')
+            r'(?s)<dt>Location</dt>\s*<dd>(.+?)<',
+            webpage, 'location', fatal=False)
 
-        formats = re.findall(
-            r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
-            webpage)
+        def extract_count(webpage, label):
+            return str_to_int(self._search_regex(
+                r'<span[^>]+class="stat-num"[^>]*>([\d,.]+)</span>\s*<span[^>]+class="stat-label"[^>]*>%s' % label,
+                webpage, label, fatal=False))
 
-        formats = [{'url': fmt[1], 'height': int_or_none(fmt[0])} for fmt in formats]
+        view_count = extract_count(webpage, 'Views')
+        comment_count = extract_count(webpage, 'Comments')
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
-            'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
-            'uploader_id': uploader_id,
             'upload_date': upload_date,
+            'uploader': uploader,
             'location': location,
+            'view_count': view_count,
+            'comment_count': comment_count,
             'formats': formats
         }

From 16d6973f8a9eb5a70c12d82aa40f57c2b4aa8c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Jun 2015 00:49:28 +0600
Subject: [PATCH 0945/1515] [viki] Pass session token around (#6005)

---
 youtube_dl/extractor/viki.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 52d10d242..51cdc6b65 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -28,11 +28,15 @@ class VikiBaseIE(InfoExtractor):
 
     _NETRC_MACHINE = 'viki'
 
+    _token = None
+
     def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
             timestamp = int(time.time())
         query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        if self._token:
+            query += '&token=%s' % self._token
         sig = hmac.new(
             self._APP_SECRET.encode('ascii'),
             query.encode('ascii'),
@@ -76,10 +80,14 @@ class VikiBaseIE(InfoExtractor):
             'password': password,
         }
 
-        self._call_api(
+        login = self._call_api(
             'sessions.json', None,
             'Logging in as %s' % username, post_data=login_form)
 
+        self._token = login.get('token')
+        if not self._token:
+            self.report_warning('Unable to get session token, login has probably failed')
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'

From 10273d6e0846cd8f3762e3777712d5cd2a0cafcd Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:22:13 -0500
Subject: [PATCH 0946/1515] toss new stuff into old file

---
 youtube_dl/extractor/__init__.py |   3 +-
 youtube_dl/extractor/bbccouk.py  | 101 ++++++++++++++++++++++++++++
 youtube_dl/extractor/bbcnews.py  | 109 -------------------------------
 3 files changed, 102 insertions(+), 111 deletions(-)
 delete mode 100644 youtube_dl/extractor/bbcnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 51d2d20e9..f9f7bdfaf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -35,8 +35,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE
-from .bbcnews import BBCNewsIE
+from .bbccouk import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index dcc5fc2fa..ea682fb6f 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -5,9 +5,11 @@ import xml.etree.ElementTree
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    parse_duration,
     int_or_none,
 )
 from ..compat import compat_HTTPError
+import re
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -394,3 +396,102 @@ class BBCCoUkIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class BBCNewsIE(BBCCoUkIE):
+    IE_NAME = 'bbc.com'
+    IE_DESC = 'BBC news'
+    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+
+    _TESTS = [{
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+        },
+        'playlist_count': 2,
+    },{
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+        },
+        'playlist_count': 9,
+    },{
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'mp4',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+
+        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
+        if pubdate:
+           pubdate = pubdate.replace('-','')
+
+        ret = []
+        # works with bbc.com/news/something-something-123456 articles
+        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
+        if not matches:
+           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+           # in http://www.bbc.com/news/video_and_audio/international
+           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+        if not matches:
+           raise ExtractorError('No video found', expected=True)
+
+        for ent in matches:
+            jent = self._parse_json(ent,list_id)
+
+            programme_id = jent.get('externalId',None)
+            xml_url = jent.get('href', None)
+
+            title = jent['caption']
+            duration = parse_duration(jent.get('duration',None))
+            description = list_title + ' - ' + jent.get('caption','')
+            thumbnail = None
+            if jent.has_key('image'):
+               thumbnail=jent['image'].get('href',None)
+
+            if programme_id:
+               formats, subtitles = self._download_media_selector(programme_id)
+            elif xml_url:
+               # Cheap fallback
+               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+               formats, subtitles = self._download_media_selector(programme_id)
+            else:
+               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+               
+            self._sort_formats(formats)
+
+            ret.append( {
+                'id': programme_id,
+                'uploader': 'BBC News',
+                'upload_date': pubdate,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            } )
+
+        if len(ret) > 0:
+           return self.playlist_result(ret, list_id, list_title)
+        raise ExtractorError('No video found', expected=True)
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
deleted file mode 100644
index fd4a5e38f..000000000
--- a/youtube_dl/extractor/bbcnews.py
+++ /dev/null
@@ -1,109 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    parse_duration,
-    int_or_none,
-)
-from ..compat import compat_HTTPError
-import re
-from .bbccouk import BBCCoUkIE
-
-class BBCNewsIE(BBCCoUkIE):
-    IE_NAME = 'bbc.com'
-    IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
-
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-
-    _TESTS = [{
-        'url': 'http://www.bbc.com/news/world-europe-32668511',
-        'info_dict': {
-            'id': 'world-europe-32668511',
-            'title': 'Russia stages massive WW2 parade despite Western boycott',
-        },
-        'playlist_count': 2,
-    },{
-        'url': 'http://www.bbc.com/news/business-28299555',
-        'info_dict': {
-            'id': 'business-28299555',
-            'title': 'Farnborough Airshow: Video highlights',
-        },
-        'playlist_count': 9,
-    },{
-        'url': 'http://www.bbc.com/news/world-europe-32041533',
-        'note': 'Video',
-        'info_dict': {
-            'id': 'p02mprgb',
-            'ext': 'mp4',
-            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'duration': 47,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }]
-
-    def _real_extract(self, url):
-        list_id = self._match_id(url)
-        webpage = self._download_webpage(url, list_id)
-
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
-
-        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
-        if pubdate:
-           pubdate = pubdate.replace('-','')
-
-        ret = []
-        # works with bbc.com/news/something-something-123456 articles
-        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
-        if not matches:
-           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-           # in http://www.bbc.com/news/video_and_audio/international
-           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-        if not matches:
-           raise ExtractorError('No video found', expected=True)
-
-        for ent in matches:
-            jent = self._parse_json(ent,list_id)
-
-            programme_id = jent.get('externalId',None)
-            xml_url = jent.get('href', None)
-
-            title = jent['caption']
-            duration = parse_duration(jent.get('duration',None))
-            description = list_title + ' - ' + jent.get('caption','')
-            thumbnail = None
-            if jent.has_key('image'):
-               thumbnail=jent['image'].get('href',None)
-
-            if programme_id:
-               formats, subtitles = self._download_media_selector(programme_id)
-            elif xml_url:
-               # Cheap fallback
-               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-               formats, subtitles = self._download_media_selector(programme_id)
-            else:
-               raise ExtractorError('data-media-meta entry has no externalId or href value.')
-               
-            self._sort_formats(formats)
-
-            ret.append( {
-                'id': programme_id,
-                'uploader': 'BBC News',
-                'upload_date': pubdate,
-                'title': title,
-                'description': description,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'formats': formats,
-                'subtitles': subtitles,
-            } )
-
-        if len(ret) > 0:
-           return self.playlist_result(ret, list_id, list_title)
-        raise ExtractorError('No video found', expected=True)

From 75ab0ebcf593ec91a46d83e69854ffa313d33309 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:24:02 -0500
Subject: [PATCH 0947/1515] no .get('..',None)

---
 youtube_dl/extractor/bbccouk.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index ea682fb6f..de4d7f9c0 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -457,15 +457,15 @@ class BBCNewsIE(BBCCoUkIE):
         for ent in matches:
             jent = self._parse_json(ent,list_id)
 
-            programme_id = jent.get('externalId',None)
-            xml_url = jent.get('href', None)
+            programme_id = jent.get('externalId')
+            xml_url = jent.get('href')
 
             title = jent['caption']
-            duration = parse_duration(jent.get('duration',None))
+            duration = parse_duration(jent.get('duration')
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):
-               thumbnail=jent['image'].get('href',None)
+               thumbnail=jent['image'].get('href')
 
             if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)

From 77c975f536befbe89bf718e86282958d391d9ffe Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:28:14 -0500
Subject: [PATCH 0948/1515] typofix

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index de4d7f9c0..f9404f3fa 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -461,7 +461,7 @@ class BBCNewsIE(BBCCoUkIE):
             xml_url = jent.get('href')
 
             title = jent['caption']
-            duration = parse_duration(jent.get('duration')
+            duration = parse_duration(jent.get('duration'))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):

From de939d89eb83c851c6db66933e5fc0c401a1a679 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:04:46 -0500
Subject: [PATCH 0949/1515] Support BBC news in other languages,
 non-mediaselector videos

---
 youtube_dl/extractor/bbccouk.py | 87 +++++++++++++++++++++++++++------
 1 file changed, 73 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f9404f3fa..72e20857b 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -401,7 +401,7 @@ class BBCCoUkIE(InfoExtractor):
 class BBCNewsIE(BBCCoUkIE):
     IE_NAME = 'bbc.com'
     IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.com/.+?/(?P<id>[^/]+)$'
 
     mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
 
@@ -432,56 +432,115 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    },{
+        'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'NA',
+            'ext': 'mp4',
+            'title': 'YPG - Tel Abyad..n tamam. kontrol.m.zde',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    },{
+        'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
+        'note': 'Video',
+        'info_dict': {
+            'id': '39275083',
+            'ext': 'mp4',
+            'title': 'Honduras militariza sus hospitales por nuevo esc.ndalo de corrupci.n',
+            'duration': 87,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
 
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'list title')
 
         pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
         if pubdate:
            pubdate = pubdate.replace('-','')
 
         ret = []
+        jsent = []
+
         # works with bbc.com/news/something-something-123456 articles
-        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
-        if not matches:
+        jsent = map(
+           lambda m: self._parse_json(m,list_id),
+           re.findall(r"data-media-meta='({[^']+})'", webpage)
+        )
+
+        if len(jsent) == 0:
+           # http://www.bbc.com/news/video_and_audio/international
+           # and single-video articles
+           masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
+           if masset:
+              jmasset = self._parse_json(masset,list_id)
+              for key, val in jmasset.get('videos',{}).items():
+                  for skey, sval in val.items():
+                      sval['id'] = skey
+                      jsent.append(sval)
+
+        if len(jsent) == 0:
            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
            # in http://www.bbc.com/news/video_and_audio/international
-           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-        if not matches:
+           # prone to breaking if entries have sourceFiles list
+           jsent = map(
+               lambda m: self._parse_json(m,list_id),
+               re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+           )          
+
+        if len(jsent) == 0:
            raise ExtractorError('No video found', expected=True)
 
-        for ent in matches:
-            jent = self._parse_json(ent,list_id)
-
+        for jent in jsent:
             programme_id = jent.get('externalId')
-            xml_url = jent.get('href')
+            xml_url = jent.get('hxref')
+
+            title = jent.get('caption',list_title)
 
-            title = jent['caption']
             duration = parse_duration(jent.get('duration'))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):
                thumbnail=jent['image'].get('href')
 
+            formats = []
+            subtitles = []
+
             if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)
+            elif jent.has_key('sourceFiles'):
+               # mediaselector not used at
+               # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
+               for key, val in jent['sourceFiles'].items():
+                  formats.append( {
+                     'ext': val.get('encoding'),
+                     'url': val.get('url'),
+                     'filesize': int(val.get('filesize')),
+                     'format_id': key
+                  } )
             elif xml_url:
                # Cheap fallback
                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
                formats, subtitles = self._download_media_selector(programme_id)
-            else:
-               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+
+            if len(formats) == 0:
+               raise ExtractorError('unsupported json media entry.\n    '+str(jent)+'\n')
                
             self._sort_formats(formats)
 
             ret.append( {
-                'id': programme_id,
+                'id': jent.get('programme_id',jent.get('id')),
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From 7bb23aeca4e9076528e3d31d501a9a288dcd444c Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:08:13 -0500
Subject: [PATCH 0950/1515] rename bbccouk.py -> bbc.py

---
 youtube_dl/extractor/__init__.py            | 2 +-
 youtube_dl/extractor/{bbccouk.py => bbc.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{bbccouk.py => bbc.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f9f7bdfaf..a48346e60 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -35,7 +35,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE, BBCNewsIE
+from .bbc import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbc.py
similarity index 100%
rename from youtube_dl/extractor/bbccouk.py
rename to youtube_dl/extractor/bbc.py

From 2a282a3b5f366ba0569bae477d5060329ba254fb Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:11:41 -0500
Subject: [PATCH 0951/1515] Unbreak breakage that was broken to test breakage

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 72e20857b..310db9d1d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -502,7 +502,7 @@ class BBCNewsIE(BBCCoUkIE):
 
         for jent in jsent:
             programme_id = jent.get('externalId')
-            xml_url = jent.get('hxref')
+            xml_url = jent.get('href')
 
             title = jent.get('caption',list_title)
 

From 964afd0689bdd7140b8ab182273d6379fe7b0548 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 03:12:17 +0800
Subject: [PATCH 0952/1515] [xvideos] Support lower-quality formats found on
 Android

Closes #5968
---
 youtube_dl/extractor/xvideos.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 2a45dc574..d8415bed4 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -5,10 +5,12 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     clean_html,
     ExtractorError,
+    determine_ext,
 )
 
 
@@ -25,6 +27,8 @@ class XVideosIE(InfoExtractor):
         }
     }
 
+    _ANDROID_USER_AGENT = 'Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -40,9 +44,30 @@ class XVideosIE(InfoExtractor):
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
 
+        formats = [{
+            'url': video_url,
+        }]
+
+        android_req = compat_urllib_request.Request(url)
+        android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
+        android_webpage = self._download_webpage(android_req, video_id, fatal=False)
+
+        if android_webpage is not None:
+            player_params_str = self._search_regex(
+                'mobileReplacePlayerDivTwoQual\(([^)]+)\)',
+                android_webpage, 'player parameters', default='')
+            player_params = list(map(lambda s: s.strip(' \''), player_params_str.split(',')))
+            if player_params:
+                formats.extend([{
+                    'url': param,
+                    'preference': -10,
+                } for param in player_params if determine_ext(param) == 'mp4'])
+
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': video_title,
             'ext': 'flv',
             'thumbnail': video_thumbnail,

From c9ac7fa909fb969ac21a6d168d09803119b018c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 04:17:54 +0600
Subject: [PATCH 0953/1515] [imdb] Fix extraction

---
 youtube_dl/extractor/imdb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f29df36b5..4bb574cf3 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -46,7 +46,7 @@ class ImdbIE(InfoExtractor):
             format_info = info['videoPlayerObject']['video']
             formats.append({
                 'format_id': f_id,
-                'url': format_info['url'],
+                'url': format_info['videoInfoList'][0]['videoUrl'],
             })
 
         return {

From af1fa6234e9623a1a09e66c24c0dedc6996645c9 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 21 Jun 2015 11:12:31 +0800
Subject: [PATCH 0954/1515] [neteasemusic] Add new extractor for music.163.com

---
 youtube_dl/extractor/__init__.py     |   9 +
 youtube_dl/extractor/neteasemusic.py | 434 +++++++++++++++++++++++++++
 2 files changed, 443 insertions(+)
 create mode 100644 youtube_dl/extractor/neteasemusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c548d8e9..c8692ffa9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -339,6 +339,15 @@ from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .nerdist import NerdistIE
+from .neteasemusic import (
+    NetEaseMusicIE,
+    NetEaseMusicAlbumIE,
+    NetEaseMusicSingerIE,
+    NetEaseMusicListIE,
+    NetEaseMusicMvIE,
+    NetEaseMusicProgramIE,
+    NetEaseMusicDjRadioIE,
+)
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nextmedia import (
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
new file mode 100644
index 000000000..ebed5d3b4
--- /dev/null
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -0,0 +1,434 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from hashlib import md5
+from datetime import datetime
+import itertools
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+
+
+class NetEaseMusicBaseIE(InfoExtractor):
+    _FORMATS = ['bMusic', 'mMusic', 'hMusic']
+    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
+    _API_BASE = 'http://music.163.com/api/'
+
+    @classmethod
+    def _encrypt(cls, dfsid):
+        salt_bytes = bytearray(str(cls._NETEASE_SALT))
+        string_bytes = bytearray(str(dfsid))
+        salt_len = len(salt_bytes)
+        for i in xrange(len(string_bytes)):
+            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
+        m = md5()
+        m.update(string_bytes)
+        result = m.digest().encode('base64')[:-1]
+        return result.replace('/', '_').replace('+', '-')
+
+    @classmethod
+    def extract_formats(cls, info):
+        formats = []
+        for song_format in cls._FORMATS:
+            details = info.get(song_format)
+            if not details:
+                continue
+            formats.append({
+                'url': 'http://m1.music.126.net/%s/%s.%s' %
+                       (cls._encrypt(details['dfsId']), details['dfsId'],
+                        details['extension']),
+                'ext': details['extension'],
+                'abr': details['bitrate'] / 1000,
+                'preference': details['bitrate'],
+                'format_id': song_format,
+                'filesize': details['size'],
+                'asr': details['sr']
+            })
+        return formats
+
+    def query_api(self, endpoint, video_id, note):
+        req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+        req.add_header('Referer', self._API_BASE)
+        return self._download_json(req, video_id, note)
+
+
+class NetEaseMusicIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:song'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/song?id=32102397',
+        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
+        'info_dict': {
+            'id': '32102397',
+            'ext': 'mp3',
+            'title': 'Bad Blood (feat. Kendrick Lamar)',
+            'creator': 'Taylor Swift / Kendrick Lamar',
+            'upload_date': '20150517',
+            'timestamp': 1431878400,
+            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
+        },
+    }, {
+        'note': 'No lyrics translation.',
+        'url': 'http://music.163.com/#/song?id=29822014',
+        'info_dict': {
+            'id': '29822014',
+            'ext': 'mp3',
+            'title': '听见下雨的声音',
+            'creator': '周杰伦',
+            'upload_date': '20141225',
+            'timestamp': 1419523200,
+            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
+        },
+    }, {
+        'note': 'No lyrics.',
+        'url': 'http://music.163.com/song?id=17241424',
+        'info_dict': {
+            'id': '17241424',
+            'ext': 'mp3',
+            'title': 'Opus 28',
+            'creator': 'Dustin O\'Halloran',
+            'upload_date': '20080211',
+            'timestamp': 1202745600,
+        },
+    }]
+
+    def _process_lyrics(self, lyrics_info):
+        original = lyrics_info.get('lrc', {}).get('lyric')
+        translated = lyrics_info.get('tlyric', {}).get('lyric')
+
+        if not translated:
+            return original
+
+        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
+        original_ts_texts = re.findall(lyrics_expr, original)
+        translation_ts_dict = {
+            time_stamp: text for time_stamp, text in re.findall(lyrics_expr, translated)
+        }
+
+        lyrics = '\n'.join([
+            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
+            for time_stamp, text in original_ts_texts
+        ])
+        return lyrics
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+
+        params = {
+            'id': song_id,
+            'ids': '[%s]' % song_id
+        }
+        info = self.query_api(
+            'song/detail?' + compat_urllib_parse.urlencode(params),
+            song_id, 'Downloading song info')['songs'][0]
+
+        formats = self.extract_formats(info)
+        self._sort_formats(formats)
+
+        lyrics_info = self.query_api(
+            'song/lyric?id=%s&lv=-1&tv=-1' % song_id, 
+            song_id, 'Downloading lyrics data')
+        lyrics = self._process_lyrics(lyrics_info)
+
+        alt_title = None
+        if info.get('alias'):
+            alt_title = '/'.join(info.get('alias'))
+
+        return {
+            'id': song_id,
+            'title': info['name'],
+            'alt_title': alt_title,
+            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
+            'timestamp': int(info.get('album', {}).get('publishTime')/1000),
+            'thumbnail': info.get('album', {}).get('picUrl'),
+            'duration': int(info.get('duration', 0)/1000),
+            'description': lyrics,
+            'formats': formats,
+        }
+
+
+class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:album'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/album?id=220780',
+        'info_dict': {
+            'id': '220780',
+            'title': 'B\'day',
+        },
+        'playlist_count': 23,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        info = self.query_api(
+            'album/%s?id=%s' % (album_id, album_id), 
+            album_id, 'Downloading album data')['album']
+
+        name = info['name']
+        desc = info.get('description')
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['songs']
+        ]
+        return self.playlist_result(entries, album_id, name, desc)
+
+
+class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:singer'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'note': 'Singer has aliases.',
+        'url': 'http://music.163.com/#/artist?id=10559',
+        'info_dict': {
+            'id': '10559',
+            'title': '张惠妹 - aMEI;阿密特',
+        },
+        'playlist_count': 50,
+    }, {
+        'note': 'Singer has translated name.',
+        'url': 'http://music.163.com/#/artist?id=124098',
+        'info_dict': {
+            'id': '124098',
+            'title': '李昇基 - 이승기',
+        },
+        'playlist_count': 50,
+    }]
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+
+        info = self.query_api(
+            'artist/%s?id=%s' % (singer_id, singer_id), 
+            singer_id, 'Downloading singer data')
+
+        name = info['artist']['name']
+        if info['artist']['trans']:
+            name = '%s - %s' % (name, info['artist']['trans'])
+        if info['artist']['alias']:
+            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['hotSongs']
+        ]
+        return self.playlist_result(entries, singer_id, name)
+
+
+class NetEaseMusicListIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:playlist'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/playlist?id=79177352',
+        'info_dict': {
+            'id': '79177352',
+            'title': 'Billboard 2007 Top 100',
+            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
+        },
+        'playlist_count': 99,
+    }, {
+        'note': 'Toplist/Charts sample',
+        'url': 'http://music.163.com/#/discover/toplist?id=3733003',
+        'info_dict': {
+            'id': '3733003',
+            'title': 're:韩国Melon排行榜周榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
+            'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
+        },
+        'playlist_count': 50,
+    }]
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        info = self.query_api(
+            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id, 
+            list_id, 'Downloading playlist data')['result']
+
+        name = info['name']
+        desc = info.get('description')
+
+        if info.get('specialType') == 10:  # is a chart/toplist
+            datestamp = datetime.fromtimestamp(info['updateTime']/1000).strftime('%Y-%m-%d')
+            name = '%s %s' % (name, datestamp)
+
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['tracks']
+        ]
+        return self.playlist_result(entries, list_id, name, desc)
+
+
+class NetEaseMusicMvIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:mv'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/mv?id=415350',
+        'info_dict': {
+            'id': '415350',
+            'ext': 'mp4',
+            'title': '이럴거면 그러지말지',
+            'description': '白雅言自作曲唱甜蜜爱情',
+            'creator': '白雅言',
+            'upload_date': '20150520',
+        },
+    }
+
+    def _real_extract(self, url):
+        mv_id = self._match_id(url)
+
+        info = self.query_api(
+            'mv/detail?id=%s&type=mp4' % mv_id,
+            mv_id, 'Downloading mv info')['data']
+
+        formats = [
+            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'preference': int(brs)}
+            for brs, mv_url in info['brs'].items()
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': mv_id,
+            'title': info['name'],
+            'description': info.get('desc') or info.get('briefDesc'),
+            'creator': info['artistName'],
+            'upload_date': info['publishTime'].replace('-', ''),
+            'formats': formats,
+            'thumbnail': info.get('cover'),
+            'duration': int(info.get('duration', 0)/1000),
+        }
+
+
+class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:program'
+    _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/program?id=10109055',
+        'info_dict': {
+            'id': '10109055',
+            'ext': 'mp3',
+            'title': '不丹足球背后的故事',
+            'description': '喜马拉雅人的足球梦 ...',
+            'creator': '大话西藏',
+            'timestamp': 1434179341,
+            'upload_date': '20150613',
+            'duration': 900,
+        },
+    }, {
+        'note': 'This program has accompanying songs.',
+        'url': 'http://music.163.com/#/program?id=10141022',
+        'info_dict': {
+            'id': '10141022',
+            'title': '25岁，你是自在如风的少年<27°C>',
+            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+        },
+        'playlist_count': 4,
+    }, {
+        'note': 'This program has accompanying songs.',
+        'url': 'http://music.163.com/#/program?id=10141022',
+        'info_dict': {
+            'id': '10141022',
+            'ext': 'mp3',
+            'title': '25岁，你是自在如风的少年<27°C>',
+            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+            'timestamp': 1434450840,
+            'upload_date': '20150616',
+        },
+        'params': {
+            'noplaylist': True
+        }
+    }]
+
+    def _real_extract(self, url):
+        program_id = self._match_id(url)
+
+        info = self.query_api(
+            'dj/program/detail?id=%s' % program_id,
+            program_id, 'Downloading program info')['program']
+
+        name = info['name']
+        description = info['description']
+
+        if not info['songs'] or self._downloader.params.get('noplaylist'):
+            if info['songs']:
+                self.to_screen(
+                    'Downloading just the main audio %s because of --no-playlist'
+                    % info['mainSong']['id'])
+            
+            formats = self.extract_formats(info['mainSong'])
+            self._sort_formats(formats)
+
+            return {
+                'id': program_id,
+                'title': name,
+                'description': description,
+                'creator': info['dj']['brand'],
+                'timestamp': int(info['createTime']/1000),
+                'thumbnail': info['coverUrl'],
+                'duration': int(info.get('duration', 0)/1000),
+                'formats': formats,
+            }
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to just download the main audio %s'
+            % (program_id, info['mainSong']['id']))
+
+        song_ids = [info['mainSong']['id']]
+        song_ids.extend([song['id'] for song in info['songs']])
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
+                            'NetEaseMusic', song_id)
+            for song_id in song_ids
+        ]
+        return self.playlist_result(entries, program_id, name, description)
+
+
+class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:djradio'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/djradio?id=42',
+        'info_dict': {
+            'id': '42',
+            'title': '声音蔓延',
+            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
+        },
+        'playlist_mincount': 40,
+    }
+    _PAGE_SIZE = 1000
+
+    def _real_extract(self, url):
+        dj_id = self._match_id(url)
+
+        name = None
+        desc = None
+        entries = []
+        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+            info = self.query_api(
+                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d' 
+                % (self._PAGE_SIZE, dj_id, offset),
+                dj_id, 'Downloading dj programs - %d' % offset)
+
+            entries.extend([
+                self.url_result(
+                    'http://music.163.com/#/program?id=%s' % program['id'],
+                    'NetEaseMusicProgram', program['id'])
+                for program in info['programs']
+            ])
+
+            if name is None:
+                radio = info['programs'][0]['radio']
+                name = radio['name']
+                desc = radio['desc']
+
+            if not info['more']:
+                break
+
+        return self.playlist_result(entries, dj_id, name, desc)

From 2da0cad6ae08e3c78f85a340260aef80a464cb20 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 21 Jun 2015 11:44:50 +0800
Subject: [PATCH 0955/1515] [neteasemusic] Do proper rounding conversion of
 millisecond timestamps/durations

---
 youtube_dl/extractor/neteasemusic.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index ebed5d3b4..a70c65ca5 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -50,6 +50,10 @@ class NetEaseMusicBaseIE(InfoExtractor):
             })
         return formats
 
+    @classmethod
+    def convert_milliseconds(cls, ms):
+        return int(round(ms/1000.0))
+
     def query_api(self, endpoint, video_id, note):
         req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
         req.add_header('Referer', self._API_BASE)
@@ -143,9 +147,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': info['name'],
             'alt_title': alt_title,
             'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
-            'timestamp': int(info.get('album', {}).get('publishTime')/1000),
+            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
             'thumbnail': info.get('album', {}).get('picUrl'),
-            'duration': int(info.get('duration', 0)/1000),
+            'duration': self.convert_milliseconds(info.get('duration', 0)),
             'description': lyrics,
             'formats': formats,
         }
@@ -255,7 +259,8 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         desc = info.get('description')
 
         if info.get('specialType') == 10:  # is a chart/toplist
-            datestamp = datetime.fromtimestamp(info['updateTime']/1000).strftime('%Y-%m-%d')
+            datestamp = datetime.fromtimestamp(
+                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
             name = '%s %s' % (name, datestamp)
 
         entries = [
@@ -302,7 +307,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             'upload_date': info['publishTime'].replace('-', ''),
             'formats': formats,
             'thumbnail': info.get('cover'),
-            'duration': int(info.get('duration', 0)/1000),
+            'duration': self.convert_milliseconds(info.get('duration', 0)),
         }
 
 
@@ -317,7 +322,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'title': '不丹足球背后的故事',
             'description': '喜马拉雅人的足球梦 ...',
             'creator': '大话西藏',
-            'timestamp': 1434179341,
+            'timestamp': 1434179342,
             'upload_date': '20150613',
             'duration': 900,
         },
@@ -338,7 +343,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'ext': 'mp3',
             'title': '25岁，你是自在如风的少年<27°C>',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
-            'timestamp': 1434450840,
+            'timestamp': 1434450841,
             'upload_date': '20150616',
         },
         'params': {
@@ -370,9 +375,9 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
                 'title': name,
                 'description': description,
                 'creator': info['dj']['brand'],
-                'timestamp': int(info['createTime']/1000),
+                'timestamp': self.convert_milliseconds(info['createTime']),
                 'thumbnail': info['coverUrl'],
-                'duration': int(info.get('duration', 0)/1000),
+                'duration': self.convert_milliseconds(info.get('duration', 0)),
                 'formats': formats,
             }
 

From 6a745c2c0fa2d627b46f2d4d8013fa69276c4fac Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:22:19 +0800
Subject: [PATCH 0956/1515] [pinkbike] PEP8

---
 youtube_dl/extractor/pinkbike.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 745433b48..a52210fab 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -44,7 +44,7 @@ class PinkbikeIE(InfoExtractor):
 
         formats = []
         for _, format_id, src in re.findall(
-            r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
+                r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
             height = int_or_none(self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             formats.append({

From b407e173e44041b1a92fb61e316f92d19834a40a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:23:58 +0800
Subject: [PATCH 0957/1515] [vimeo/generic] Move detection logic from GenericIE
 to VimeoIE

---
 youtube_dl/extractor/generic.py | 16 ++++------------
 youtube_dl/extractor/vimeo.py   | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f6b984300..bf689f531 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -43,6 +43,7 @@ from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
+from .vimeo import VimeoIE
 
 
 class GenericIE(InfoExtractor):
@@ -1089,18 +1090,9 @@ class GenericIE(InfoExtractor):
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')
 
-        # Look for embedded (iframe) Vimeo player
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
-        if mobj:
-            player_url = unescapeHTML(mobj.group('url'))
-            surl = smuggle_url(player_url, {'Referer': url})
-            return self.url_result(surl)
-        # Look for embedded (swf embed) Vimeo player
-        mobj = re.search(
-            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
-        if mobj:
-            return self.url_result(mobj.group(1))
+        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
+        if vimeo_url is not None:
+            return self.url_result(vimeo_url)
 
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index f300c7ca4..cae90205d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -22,6 +22,7 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     urlencode_postdata,
+    unescapeHTML,
 )
 
 
@@ -173,6 +174,21 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
     ]
 
+    @staticmethod
+    def _extract_vimeo_url(url, webpage):
+        # Look for embedded (iframe) Vimeo player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
+        if mobj:
+            player_url = unescapeHTML(mobj.group('url'))
+            surl = smuggle_url(player_url, {'Referer': url})
+            return surl
+        # Look for embedded (swf embed) Vimeo player
+        mobj = re.search(
+            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
+        if mobj:
+            return mobj.group(1)
+
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:

From c5895d5dbdc33fbad1c91f448704d7711448220d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:30:38 +0800
Subject: [PATCH 0958/1515] [tumblr] Support Vimeo embeds (fixes #5969)

---
 youtube_dl/extractor/tumblr.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 63c20310d..9ead13a91 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from .pornhub import PornHubIE
+from .vimeo import VimeoIE
 
 
 class TumblrIE(InfoExtractor):
@@ -40,6 +41,17 @@ class TumblrIE(InfoExtractor):
             'timestamp': 1430931613,
         },
         'add_ie': ['Vidme'],
+    }, {
+        'url': 'http://camdamage.tumblr.com/post/98846056295/',
+        'md5': 'a9e0c8371ea1ca306d6554e3fecf50b6',
+        'info_dict': {
+            'id': '105463834',
+            'ext': 'mp4',
+            'title': 'Cam Damage-HD 720p',
+            'uploader': 'John Moyer',
+            'uploader_id': 'user32021558',
+        },
+        'add_ie': ['Vimeo'],
     }]
 
     def _real_extract(self, url):
@@ -60,6 +72,10 @@ class TumblrIE(InfoExtractor):
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
 
+        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
+        if vimeo_url:
+            return self.url_result(vimeo_url, 'Vimeo')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From 396726244a9096f142f5420ba5f3a1a36abb9a86 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:53:17 +0800
Subject: [PATCH 0959/1515] [utils/ffmpeg] Move ISO 639 related codes to utils

---
 youtube_dl/postprocessor/ffmpeg.py | 196 +---------------------------
 youtube_dl/utils.py                | 202 +++++++++++++++++++++++++++++
 2 files changed, 204 insertions(+), 194 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index cc65b34e7..fe7e0a8ee 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -21,6 +21,7 @@ from ..utils import (
     shell_quote,
     subtitles_filename,
     dfxp2srt,
+    ISO639Utils,
 )
 
 
@@ -307,199 +308,6 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
-    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-    _lang_map = {
-        'aa': 'aar',
-        'ab': 'abk',
-        'ae': 'ave',
-        'af': 'afr',
-        'ak': 'aka',
-        'am': 'amh',
-        'an': 'arg',
-        'ar': 'ara',
-        'as': 'asm',
-        'av': 'ava',
-        'ay': 'aym',
-        'az': 'aze',
-        'ba': 'bak',
-        'be': 'bel',
-        'bg': 'bul',
-        'bh': 'bih',
-        'bi': 'bis',
-        'bm': 'bam',
-        'bn': 'ben',
-        'bo': 'bod',
-        'br': 'bre',
-        'bs': 'bos',
-        'ca': 'cat',
-        'ce': 'che',
-        'ch': 'cha',
-        'co': 'cos',
-        'cr': 'cre',
-        'cs': 'ces',
-        'cu': 'chu',
-        'cv': 'chv',
-        'cy': 'cym',
-        'da': 'dan',
-        'de': 'deu',
-        'dv': 'div',
-        'dz': 'dzo',
-        'ee': 'ewe',
-        'el': 'ell',
-        'en': 'eng',
-        'eo': 'epo',
-        'es': 'spa',
-        'et': 'est',
-        'eu': 'eus',
-        'fa': 'fas',
-        'ff': 'ful',
-        'fi': 'fin',
-        'fj': 'fij',
-        'fo': 'fao',
-        'fr': 'fra',
-        'fy': 'fry',
-        'ga': 'gle',
-        'gd': 'gla',
-        'gl': 'glg',
-        'gn': 'grn',
-        'gu': 'guj',
-        'gv': 'glv',
-        'ha': 'hau',
-        'he': 'heb',
-        'hi': 'hin',
-        'ho': 'hmo',
-        'hr': 'hrv',
-        'ht': 'hat',
-        'hu': 'hun',
-        'hy': 'hye',
-        'hz': 'her',
-        'ia': 'ina',
-        'id': 'ind',
-        'ie': 'ile',
-        'ig': 'ibo',
-        'ii': 'iii',
-        'ik': 'ipk',
-        'io': 'ido',
-        'is': 'isl',
-        'it': 'ita',
-        'iu': 'iku',
-        'ja': 'jpn',
-        'jv': 'jav',
-        'ka': 'kat',
-        'kg': 'kon',
-        'ki': 'kik',
-        'kj': 'kua',
-        'kk': 'kaz',
-        'kl': 'kal',
-        'km': 'khm',
-        'kn': 'kan',
-        'ko': 'kor',
-        'kr': 'kau',
-        'ks': 'kas',
-        'ku': 'kur',
-        'kv': 'kom',
-        'kw': 'cor',
-        'ky': 'kir',
-        'la': 'lat',
-        'lb': 'ltz',
-        'lg': 'lug',
-        'li': 'lim',
-        'ln': 'lin',
-        'lo': 'lao',
-        'lt': 'lit',
-        'lu': 'lub',
-        'lv': 'lav',
-        'mg': 'mlg',
-        'mh': 'mah',
-        'mi': 'mri',
-        'mk': 'mkd',
-        'ml': 'mal',
-        'mn': 'mon',
-        'mr': 'mar',
-        'ms': 'msa',
-        'mt': 'mlt',
-        'my': 'mya',
-        'na': 'nau',
-        'nb': 'nob',
-        'nd': 'nde',
-        'ne': 'nep',
-        'ng': 'ndo',
-        'nl': 'nld',
-        'nn': 'nno',
-        'no': 'nor',
-        'nr': 'nbl',
-        'nv': 'nav',
-        'ny': 'nya',
-        'oc': 'oci',
-        'oj': 'oji',
-        'om': 'orm',
-        'or': 'ori',
-        'os': 'oss',
-        'pa': 'pan',
-        'pi': 'pli',
-        'pl': 'pol',
-        'ps': 'pus',
-        'pt': 'por',
-        'qu': 'que',
-        'rm': 'roh',
-        'rn': 'run',
-        'ro': 'ron',
-        'ru': 'rus',
-        'rw': 'kin',
-        'sa': 'san',
-        'sc': 'srd',
-        'sd': 'snd',
-        'se': 'sme',
-        'sg': 'sag',
-        'si': 'sin',
-        'sk': 'slk',
-        'sl': 'slv',
-        'sm': 'smo',
-        'sn': 'sna',
-        'so': 'som',
-        'sq': 'sqi',
-        'sr': 'srp',
-        'ss': 'ssw',
-        'st': 'sot',
-        'su': 'sun',
-        'sv': 'swe',
-        'sw': 'swa',
-        'ta': 'tam',
-        'te': 'tel',
-        'tg': 'tgk',
-        'th': 'tha',
-        'ti': 'tir',
-        'tk': 'tuk',
-        'tl': 'tgl',
-        'tn': 'tsn',
-        'to': 'ton',
-        'tr': 'tur',
-        'ts': 'tso',
-        'tt': 'tat',
-        'tw': 'twi',
-        'ty': 'tah',
-        'ug': 'uig',
-        'uk': 'ukr',
-        'ur': 'urd',
-        'uz': 'uzb',
-        've': 'ven',
-        'vi': 'vie',
-        'vo': 'vol',
-        'wa': 'wln',
-        'wo': 'wol',
-        'xh': 'xho',
-        'yi': 'yid',
-        'yo': 'yor',
-        'za': 'zha',
-        'zh': 'zho',
-        'zu': 'zul',
-    }
-
-    @classmethod
-    def _conver_lang_code(cls, code):
-        """Convert language code from ISO 639-1 to ISO 639-2/T"""
-        return cls._lang_map.get(code[:2])
-
     def run(self, information):
         if information['ext'] not in ['mp4', 'mkv']:
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
@@ -525,7 +333,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             opts += ['-c:s', 'mov_text']
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1)])
-            lang_code = self._conver_lang_code(lang)
+            lang_code = ISO639Utils.short2long(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52d198fa3..259a9d634 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1879,6 +1879,208 @@ def dfxp2srt(dfxp_data):
     return ''.join(out)
 
 
+class ISO639Utils(object):
+    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
+    _lang_map = {
+        'aa': 'aar',
+        'ab': 'abk',
+        'ae': 'ave',
+        'af': 'afr',
+        'ak': 'aka',
+        'am': 'amh',
+        'an': 'arg',
+        'ar': 'ara',
+        'as': 'asm',
+        'av': 'ava',
+        'ay': 'aym',
+        'az': 'aze',
+        'ba': 'bak',
+        'be': 'bel',
+        'bg': 'bul',
+        'bh': 'bih',
+        'bi': 'bis',
+        'bm': 'bam',
+        'bn': 'ben',
+        'bo': 'bod',
+        'br': 'bre',
+        'bs': 'bos',
+        'ca': 'cat',
+        'ce': 'che',
+        'ch': 'cha',
+        'co': 'cos',
+        'cr': 'cre',
+        'cs': 'ces',
+        'cu': 'chu',
+        'cv': 'chv',
+        'cy': 'cym',
+        'da': 'dan',
+        'de': 'deu',
+        'dv': 'div',
+        'dz': 'dzo',
+        'ee': 'ewe',
+        'el': 'ell',
+        'en': 'eng',
+        'eo': 'epo',
+        'es': 'spa',
+        'et': 'est',
+        'eu': 'eus',
+        'fa': 'fas',
+        'ff': 'ful',
+        'fi': 'fin',
+        'fj': 'fij',
+        'fo': 'fao',
+        'fr': 'fra',
+        'fy': 'fry',
+        'ga': 'gle',
+        'gd': 'gla',
+        'gl': 'glg',
+        'gn': 'grn',
+        'gu': 'guj',
+        'gv': 'glv',
+        'ha': 'hau',
+        'he': 'heb',
+        'hi': 'hin',
+        'ho': 'hmo',
+        'hr': 'hrv',
+        'ht': 'hat',
+        'hu': 'hun',
+        'hy': 'hye',
+        'hz': 'her',
+        'ia': 'ina',
+        'id': 'ind',
+        'ie': 'ile',
+        'ig': 'ibo',
+        'ii': 'iii',
+        'ik': 'ipk',
+        'io': 'ido',
+        'is': 'isl',
+        'it': 'ita',
+        'iu': 'iku',
+        'ja': 'jpn',
+        'jv': 'jav',
+        'ka': 'kat',
+        'kg': 'kon',
+        'ki': 'kik',
+        'kj': 'kua',
+        'kk': 'kaz',
+        'kl': 'kal',
+        'km': 'khm',
+        'kn': 'kan',
+        'ko': 'kor',
+        'kr': 'kau',
+        'ks': 'kas',
+        'ku': 'kur',
+        'kv': 'kom',
+        'kw': 'cor',
+        'ky': 'kir',
+        'la': 'lat',
+        'lb': 'ltz',
+        'lg': 'lug',
+        'li': 'lim',
+        'ln': 'lin',
+        'lo': 'lao',
+        'lt': 'lit',
+        'lu': 'lub',
+        'lv': 'lav',
+        'mg': 'mlg',
+        'mh': 'mah',
+        'mi': 'mri',
+        'mk': 'mkd',
+        'ml': 'mal',
+        'mn': 'mon',
+        'mr': 'mar',
+        'ms': 'msa',
+        'mt': 'mlt',
+        'my': 'mya',
+        'na': 'nau',
+        'nb': 'nob',
+        'nd': 'nde',
+        'ne': 'nep',
+        'ng': 'ndo',
+        'nl': 'nld',
+        'nn': 'nno',
+        'no': 'nor',
+        'nr': 'nbl',
+        'nv': 'nav',
+        'ny': 'nya',
+        'oc': 'oci',
+        'oj': 'oji',
+        'om': 'orm',
+        'or': 'ori',
+        'os': 'oss',
+        'pa': 'pan',
+        'pi': 'pli',
+        'pl': 'pol',
+        'ps': 'pus',
+        'pt': 'por',
+        'qu': 'que',
+        'rm': 'roh',
+        'rn': 'run',
+        'ro': 'ron',
+        'ru': 'rus',
+        'rw': 'kin',
+        'sa': 'san',
+        'sc': 'srd',
+        'sd': 'snd',
+        'se': 'sme',
+        'sg': 'sag',
+        'si': 'sin',
+        'sk': 'slk',
+        'sl': 'slv',
+        'sm': 'smo',
+        'sn': 'sna',
+        'so': 'som',
+        'sq': 'sqi',
+        'sr': 'srp',
+        'ss': 'ssw',
+        'st': 'sot',
+        'su': 'sun',
+        'sv': 'swe',
+        'sw': 'swa',
+        'ta': 'tam',
+        'te': 'tel',
+        'tg': 'tgk',
+        'th': 'tha',
+        'ti': 'tir',
+        'tk': 'tuk',
+        'tl': 'tgl',
+        'tn': 'tsn',
+        'to': 'ton',
+        'tr': 'tur',
+        'ts': 'tso',
+        'tt': 'tat',
+        'tw': 'twi',
+        'ty': 'tah',
+        'ug': 'uig',
+        'uk': 'ukr',
+        'ur': 'urd',
+        'uz': 'uzb',
+        've': 'ven',
+        'vi': 'vie',
+        'vo': 'vol',
+        'wa': 'wln',
+        'wo': 'wol',
+        'xh': 'xho',
+        'yi': 'yid',
+        'yo': 'yor',
+        'za': 'zha',
+        'zh': 'zho',
+        'zu': 'zul',
+    }
+
+    @classmethod
+    def short2long(cls, code):
+        """Convert language code from ISO 639-1 to ISO 639-2/T"""
+        return cls._lang_map.get(code[:2])
+
+    @classmethod
+    def long2short(cls, code):
+        """Convert language code from ISO 639-2/T to ISO 639-1"""
+        for short_name, long_name in cls._lang_map.items():
+            if long_name == code:
+                return short_name
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From 607841af64d308eaf577e528fd7317a8b382b8e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:55:26 +0800
Subject: [PATCH 0960/1515] [adobetv] Support embeddable videos (closes #6039)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/adobetv.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c548d8e9..0f4af88f0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -4,7 +4,10 @@ from .abc import ABCIE
 from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
-from .adobetv import AdobeTVIE
+from .adobetv import (
+    AdobeTVIE,
+    AdobeTVVideoIE,
+)
 from .adultswim import AdultSwimIE
 from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 97d128560..695a4a15c 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -5,6 +5,8 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     str_to_int,
+    float_or_none,
+    ISO639Utils,
 )
 
 
@@ -69,3 +71,60 @@ class AdobeTVIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
         }
+
+
+class AdobeTVVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'https://video.tv.adobe.com/v/2456/',
+        'md5': '43662b577c018ad707a63766462b1e87',
+        'info_dict': {
+            'id': '2456',
+            'ext': 'mp4',
+            'title': 'New experience with Acrobat DC',
+            'description': 'New experience with Acrobat DC',
+            'duration': 248.667,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player_params = self._parse_json(self._search_regex(
+            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'player parameters'),
+            video_id)
+
+        formats = [{
+            'url': source['src'],
+            'width': source.get('width'),
+            'height': source.get('height'),
+            'tbr': source.get('bitrate'),
+        } for source in player_params['sources']]
+
+        # For both metadata and downloaded files the duration varies among
+        # formats. I just pick the max one
+        duration = max(filter(None, [
+            float_or_none(source.get('duration'), scale=1000)
+            for source in player_params['sources']]))
+
+        subtitles = {}
+        for translation in player_params.get('translations', []):
+            lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
+            if lang_id not in subtitles:
+                subtitles[lang_id] = []
+            subtitles[lang_id].append({
+                'url': translation['vttPath'],
+                'ext': 'vtt',
+            })
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': player_params['title'],
+            'description': self._og_search_description(webpage),
+            'duration': duration,
+            'subtitles': subtitles,
+        }

From 4e3357717312ac56145ba166a1ae2806f6db8337 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 19:16:59 +0800
Subject: [PATCH 0961/1515] [utils] Support ttaf1 namespace in TTML

It's found in bbc.co.uk. See #6038
---
 youtube_dl/utils.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 259a9d634..a2746b2d1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1841,7 +1841,10 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
-    _x = functools.partial(xpath_with_ns, ns_map={'ttml': 'http://www.w3.org/ns/ttml'})
+    _x = functools.partial(xpath_with_ns, ns_map={
+        'ttml': 'http://www.w3.org/ns/ttml',
+        'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
+    })
 
     def parse_node(node):
         str_or_empty = functools.partial(str_or_none, default='')
@@ -1849,9 +1852,9 @@ def dfxp2srt(dfxp_data):
         out = str_or_empty(node.text)
 
         for child in node:
-            if child.tag in (_x('ttml:br'), 'br'):
+            if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
                 out += '\n' + str_or_empty(child.tail)
-            elif child.tag in (_x('ttml:span'), 'span'):
+            elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
                 out += str_or_empty(parse_node(child))
             else:
                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
@@ -1860,7 +1863,7 @@ def dfxp2srt(dfxp_data):
 
     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')

From 78294e6a9ce2c9a294d663ac79936df7353b9980 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 19:22:26 +0800
Subject: [PATCH 0962/1515] [bbccouk] Remove TTML to srt conversion codes

It's broken. See #6038
---
 youtube_dl/extractor/bbccouk.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 0305f88b5..5825d2867 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -251,26 +251,11 @@ class BBCCoUkIE(InfoExtractor):
         for connection in self._extract_connections(media):
             captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
-            ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
-            srt = ''
-
-            def _extract_text(p):
-                if p.text is not None:
-                    stripped_text = p.text.strip()
-                    if stripped_text:
-                        return stripped_text
-                return ' '.join(span.text.strip() for span in p.findall('{http://www.w3.org/2006/10/ttaf1}span'))
-            for pos, p in enumerate(ps):
-                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'), _extract_text(p))
             subtitles[lang] = [
                 {
                     'url': connection.get('href'),
                     'ext': 'ttml',
                 },
-                {
-                    'data': srt,
-                    'ext': 'srt',
-                },
             ]
         return subtitles
 

From 756f574e4e7160ca5b39c6e18ec5168beb4a8eb1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 21:30:34 +0800
Subject: [PATCH 0963/1515] [dailymotion/generic] Add DailymotionCloudIE

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/dailymotion.py | 42 +++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py     | 17 ++++++++++++
 3 files changed, 60 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0f4af88f0..bd3c3193f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
     DailymotionUserIE,
+    DailymotionCloudIE,
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 70aa4333c..96f0ed9ad 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -251,3 +251,45 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': full_user,
             'entries': self._extract_entries(user),
         }
+
+
+class DailymotionCloudIE(DailymotionBaseInfoExtractor):
+    _VALID_URL = r'http://api\.dmcloud\.net/embed/[^/]+/(?P<id>[^/?]+)'
+
+    _TEST = {
+        # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
+        # Tested at FranceTvInfo_2
+        'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
+        'only_matching': True,
+    }
+
+    @classmethod
+    def _extract_dmcloud_url(self, webpage):
+        mobj = re.search(r'<iframe[^>]+src=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        if mobj:
+            return mobj.group(1)
+
+        mobj = re.search(r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        if mobj:
+            return mobj.group(1)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        request = self._build_request(url)
+        webpage = self._download_webpage(request, video_id)
+
+        title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
+
+        video_info = self._parse_json(self._search_regex(
+            r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id)
+
+        # TODO: parse ios_url, which is in fact a manifest
+        video_url = video_info['mp4_url']
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': video_info.get('thumbnail_url'),
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bf689f531..07939b196 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -44,6 +44,7 @@ from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .vimeo import VimeoIE
+from .dailymotion import DailymotionCloudIE
 
 
 class GenericIE(InfoExtractor):
@@ -813,6 +814,17 @@ class GenericIE(InfoExtractor):
                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
                 'uploader': 'Rogers Sportsnet',
             },
+        },
+        # Dailymotion Cloud video
+        {
+            'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
+            'md5': '49444254273501a64675a7e68c502681',
+            'info_dict': {
+                'id': '5585de919473990de4bee11b',
+                'ext': 'mp4',
+                'title': 'Le débat',
+                'thumbnail': 're:^https?://.*\.jpe?g$',
+            }
         }
     ]
 
@@ -1486,6 +1498,11 @@ class GenericIE(InfoExtractor):
         if senate_isvp_url:
             return self.url_result(senate_isvp_url, 'SenateISVP')
 
+        # Look for Dailymotion Cloud videos
+        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
+        if dmcloud_url:
+            return self.url_result(dmcloud_url, 'DailymotionCloud')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 6f96e308d0fa7674ac88e1e80fc602413f9a6b31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 21:31:33 +0800
Subject: [PATCH 0964/1515] [francetvinfo.fr] Support dmcloud embeds (fixes
 #6034)

---
 youtube_dl/extractor/francetv.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index db0bbec1e..b2c984bf2 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -18,6 +18,7 @@ from ..utils import (
     parse_duration,
     determine_ext,
 )
+from .dailymotion import DailymotionCloudIE
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -131,12 +132,26 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'skip_download': 'HLS (reqires ffmpeg)'
         },
         'skip': 'Ce direct est terminé et sera disponible en rattrapage dans quelques minutes.',
+    }, {
+        'url': 'http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html',
+        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
+        'info_dict': {
+            'id': '556e03339473995ee145930c',
+            'ext': 'mp4',
+            'title': 'Les entreprises familiales : le secret de la réussite',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+        }
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
+
+        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
+        if dmcloud_url:
+            return self.url_result(dmcloud_url, 'DailymotionCloud')
+
         video_id, catalogue = self._search_regex(
             r'id-video=([^@]+@[^"]+)', webpage, 'video id').split('@')
         return self._extract_video(video_id, catalogue)

From 3f3308cd75fc068e4d67d00aa7d7892e02ab16e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 23:29:40 +0800
Subject: [PATCH 0965/1515] Revert "[sohu] Update extractor"

This reverts commit 32060c6d6b618fa858b2ce43db34d02fd43bc542.
---
 youtube_dl/extractor/sohu.py | 46 ++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 29bd9ce6f..7644cc02d 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import ExtractorError
+from ..utils import (
+    sanitize_url_path_consecutive_slashes,
+    ExtractorError,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -26,7 +29,7 @@ class SohuIE(InfoExtractor):
         'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',
+        'md5': '699060e75cf58858dd47fb9c03c42cfb',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -34,7 +37,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '49308ff6dafde5ece51137d04aec311e',
+        'md5': '9bf34be48f2f4dadcb226c74127e203c',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -48,7 +51,7 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': '492923eac023ba2f13ff69617c32754a',
+            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -56,7 +59,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': 'de604848c0e8e9c4a4dde7e1347c0637',
+            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -64,7 +67,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '93584716ee0657c0b205b8aa3d27aa13',
+            'md5': '8407e634175fdac706766481b9443450',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',
@@ -139,21 +142,24 @@ class SohuIE(InfoExtractor):
         for i in range(part_count):
             formats = []
             for format_id, format_data in formats_json.items():
-                data = format_data['data']
+                allot = format_data['allot']
+                prot = format_data['prot']
 
-                # URLs starts with http://newflv.sohu.ccgslb.net/ is not usable
-                # so retry until got a working URL
-                video_url = 'newflv.sohu.ccgslb.net'
-                retries = 0
-                while 'newflv.sohu.ccgslb.net' in video_url and retries < 5:
-                    download_note = 'Download information from CDN gateway for format ' + format_id
-                    if retries > 0:
-                        download_note += ' (retry #%d)' % retries
-                    retries += 1
-                    cdn_info = self._download_json(
-                        'http://data.vod.itc.cn/cdnList?new=' + data['su'][i],
-                        video_id, download_note)
-                    video_url = cdn_info['url']
+                data = format_data['data']
+                clips_url = data['clipsURL']
+                su = data['su']
+
+                part_str = self._download_webpage(
+                    'http://%s/?prot=%s&file=%s&new=%s' %
+                    (allot, prot, clips_url[i], su[i]),
+                    video_id,
+                    'Downloading %s video URL part %d of %d'
+                    % (format_id, i + 1, part_count))
+
+                part_info = part_str.split('|')
+
+                video_url = sanitize_url_path_consecutive_slashes(
+                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({
                     'url': video_url,

From 98ca102441624c2a1a66114c32e0142899f78dd3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Jun 2015 00:59:55 +0800
Subject: [PATCH 0966/1515] [sohu] Fix extraction again

---
 youtube_dl/extractor/sohu.py | 42 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7644cc02d..ba2d5e19b 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,10 +6,10 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_request
+    compat_urllib_request,
+    compat_urllib_parse,
 )
 from ..utils import (
-    sanitize_url_path_consecutive_slashes,
     ExtractorError,
 )
 
@@ -143,23 +143,41 @@ class SohuIE(InfoExtractor):
             formats = []
             for format_id, format_data in formats_json.items():
                 allot = format_data['allot']
-                prot = format_data['prot']
 
                 data = format_data['data']
                 clips_url = data['clipsURL']
                 su = data['su']
 
-                part_str = self._download_webpage(
-                    'http://%s/?prot=%s&file=%s&new=%s' %
-                    (allot, prot, clips_url[i], su[i]),
-                    video_id,
-                    'Downloading %s video URL part %d of %d'
-                    % (format_id, i + 1, part_count))
+                video_url = 'newflv.sohu.ccgslb.net'
+                cdnId = None
+                retries = 0
 
-                part_info = part_str.split('|')
+                while 'newflv.sohu.ccgslb.net' in video_url:
+                    params = {
+                        'prot': 9,
+                        'file': clips_url[i],
+                        'new': su[i],
+                        'prod': 'flash',
+                    }
 
-                video_url = sanitize_url_path_consecutive_slashes(
-                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
+                    if cdnId is not None:
+                        params['idc'] = cdnId
+
+                    download_note = 'Downloading %s video URL part %d of %d' % (
+                        format_id, i + 1, part_count)
+
+                    if retries > 0:
+                        download_note += ' (retry #%d)' % retries
+                    part_info = self._parse_json(self._download_webpage(
+                        'http://%s/?%s' % (allot, compat_urllib_parse.urlencode(params)),
+                        video_id, download_note), video_id)
+
+                    video_url = part_info['url']
+                    cdnId = part_info.get('nid')
+
+                    retries += 1
+                    if retries > 5:
+                        raise ExtractorError('Failed to get video URL')
 
                 formats.append({
                     'url': video_url,

From 0bbba43ed0c68b612fcafbdad460a93b733b9f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:10:38 +0600
Subject: [PATCH 0967/1515] [xhamster:embed] Add extractor (Closes #6032)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/xhamster.py | 28 +++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd3c3193f..dc1a302e6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -701,7 +701,10 @@ from .wrzuta import WrzutaIE
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
-from .xhamster import XHamsterIE
+from .xhamster import (
+    XHamsterIE,
+    XHamsterEmbedIE,
+)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 4527567f8..ae3c5962b 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -13,7 +13,6 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    """Information Extractor for xHamster"""
     _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
     _TESTS = [
         {
@@ -133,3 +132,30 @@ class XHamsterIE(InfoExtractor):
             'age_limit': age_limit,
             'formats': formats,
         }
+
+
+class XHamsterEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://xhamster.com/xembed.php?video=3328539',
+        'info_dict': {
+            'id': '3328539',
+            'ext': 'mp4',
+            'title': 'Pen Masturbation',
+            'upload_date': '20140728',
+            'uploader_id': 'anonymous',
+            'duration': 5,
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
+            webpage, 'xhamster url')
+
+        return self.url_result(video_url, 'XHamster');

From 2bb5b6d0a1671957c7a2e6d6433901b2a1b8f48f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:11:25 +0600
Subject: [PATCH 0968/1515] [generic] Add support for xhamster embeds

---
 youtube_dl/extractor/generic.py  | 6 ++++++
 youtube_dl/extractor/xhamster.py | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 07939b196..e108bde66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -43,6 +43,7 @@ from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
+from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 
@@ -1331,6 +1332,11 @@ class GenericIE(InfoExtractor):
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
 
+        # Look for embedded XHamster player
+        xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
+        if xhamster_urls:
+            return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ae3c5962b..725e01ac3 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -149,6 +149,12 @@ class XHamsterEmbedIE(InfoExtractor):
         }
     }
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?xhamster\.com/xembed\.php\?video=\d+)\1',
+            webpage)]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From c76799c5553b3b48bb7cc73dec452c3637a8670a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:18:28 +0600
Subject: [PATCH 0969/1515] [extractor/generic] Add test for xhamster embed

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e108bde66..20fcd8170 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -336,6 +336,15 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # XHamster embed
+        {
+            'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
+            'info_dict': {
+                'id': 'showthread',
+                'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
+            },
+            'playlist_mincount': 7,
+        },
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',

From a5158f38a31e863a39de8f66c26469a5d4469280 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Jun 2015 15:02:53 +0800
Subject: [PATCH 0970/1515] [generic/adobetv] Support AdobeTVVideo embeds
 (#6039)

---
 youtube_dl/extractor/adobetv.py |  1 +
 youtube_dl/extractor/generic.py | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 695a4a15c..5e43adc51 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -77,6 +77,7 @@ class AdobeTVVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
 
     _TEST = {
+        # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
         'url': 'https://video.tv.adobe.com/v/2456/',
         'md5': '43662b577c018ad707a63766462b1e87',
         'info_dict': {
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 20fcd8170..5c03fddc6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -835,6 +835,18 @@ class GenericIE(InfoExtractor):
                 'title': 'Le débat',
                 'thumbnail': 're:^https?://.*\.jpe?g$',
             }
+        },
+        # AdobeTVVideo embed
+        {
+            'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
+            'md5': '43662b577c018ad707a63766462b1e87',
+            'info_dict': {
+                'id': '2456',
+                'ext': 'mp4',
+                'title': 'New experience with Acrobat DC',
+                'description': 'New experience with Acrobat DC',
+                'duration': 248.667,
+            },
         }
     ]
 
@@ -1518,6 +1530,15 @@ class GenericIE(InfoExtractor):
         if dmcloud_url:
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
+        # Look for AdobeTVVideo embeds
+        mobj = re.search(
+            r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                self._proto_relative_url(unescapeHTML(mobj.group(1))),
+                'AdobeTVVideo')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 25701d5a2ca8c7a58c91e11c6a30d4e61b02e89c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 22 Jun 2015 11:18:52 +0200
Subject: [PATCH 0971/1515] [xhamster] pep8: remove trailing ';'

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 725e01ac3..b4ad513a0 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -164,4 +164,4 @@ class XHamsterEmbedIE(InfoExtractor):
             r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
             webpage, 'xhamster url')
 
-        return self.url_result(video_url, 'XHamster');
+        return self.url_result(video_url, 'XHamster')

From 255f5694aabe07fa7f33978c6b97ced469e172db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Jun 2015 20:11:15 +0600
Subject: [PATCH 0972/1515] [faz] Extend _VALID_URL (Closes #6050)

---
 youtube_dl/extractor/faz.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 3c39ca451..cebdd0193 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 
 class FazIE(InfoExtractor):
     IE_NAME = 'faz.net'
-    _VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?faz\.net/(?:[^/]+/)*.*?-(?P<id>\d+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
         'info_dict': {
             'id': '12610585',
@@ -16,7 +16,22 @@ class FazIE(InfoExtractor):
             'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
             'description': 'md5:1453fbf9a0d041d985a47306192ea253',
         },
-    }
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/foobarblafasel-13659345.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From a9dcf4a860214e37971ab05f27f74bbae65ff8ae Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 23 Jun 2015 01:08:07 -0500
Subject: [PATCH 0973/1515] Prefer externalId over non-mediaserver-specific
 hashkey for video id.

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 310db9d1d..fed344ea0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -540,7 +540,7 @@ class BBCNewsIE(BBCCoUkIE):
             self._sort_formats(formats)
 
             ret.append( {
-                'id': jent.get('programme_id',jent.get('id')),
+                'id': jent.get('id') if programme_id == None else programme_id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From e20d0c1e69f66a82dd493680351538ea92393fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Jun 2015 21:34:29 +0600
Subject: [PATCH 0974/1515] [brightcove] Use `compat_xml_parse_error` (Closes
 #6060)

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d768f99e6..4721c2293 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_xml_parse_error,
 )
 from ..utils import (
     determine_ext,
@@ -119,7 +120,7 @@ class BrightcoveIE(InfoExtractor):
 
         try:
             object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
-        except xml.etree.ElementTree.ParseError:
+        except compat_xml_parse_error:
             return
 
         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')

From 2fece970b80022574a6b54c936820897cfd10d70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 03:08:24 +0600
Subject: [PATCH 0975/1515] [extractor/generic] Clarify generic extraction
 warning

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3d672197c..c8582bda9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -977,10 +977,10 @@ class GenericIE(InfoExtractor):
                 'upload_date': upload_date,
             }
 
-        if (not self._downloader.params.get('test', False) and
-                not is_intentional and
-                not self._downloader.params.get('force_generic_extractor', False)):
-            self._downloader.report_warning('Falling back on generic information extractor.')
+        if not self._downloader.params.get('test', False) and not is_intentional:
+            force = self._downloader.params.get('force_generic_extractor', False)
+            self._downloader.report_warning(
+                '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
         if not full_response:
             request = compat_urllib_request.Request(url)

From 18b5e1e5348ba3a6d1b6a98e97217eebb3d32a1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Jun 2015 16:00:12 +0800
Subject: [PATCH 0976/1515] [drbonanza] Fix extraction of videos

---
 youtube_dl/extractor/drbonanza.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 7626219ba..8b98b013a 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -15,7 +15,6 @@ class DRBonanzaIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
-        'md5': 'fe330252ddea607635cf2eb2c99a0af3',
         'info_dict': {
             'id': '65517',
             'ext': 'mp4',
@@ -26,6 +25,9 @@ class DRBonanzaIE(InfoExtractor):
             'upload_date': '20110120',
             'duration': 3664,
         },
+        'params': {
+            'skip_download': True,  # requires rtmp
+        },
     }, {
         'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
         'md5': '6dfe039417e76795fb783c52da3de11d',
@@ -93,6 +95,11 @@ class DRBonanzaIE(InfoExtractor):
                         'format_id': file['Type'].replace('Video', ''),
                         'preference': preferencemap.get(file['Type'], -10),
                     })
+                    if format['url'].startswith('rtmp'):
+                        rtmp_url = format['url']
+                        format['rtmp_live'] = True  # --resume does not work
+                        if '/bonanza/' in rtmp_url:
+                            format['play_path'] = rtmp_url.split('/bonanza/')[1]
                     formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
@@ -111,9 +118,6 @@ class DRBonanzaIE(InfoExtractor):
         description = '%s\n%s\n%s\n' % (
             info['Description'], info['Actors'], info['Colophon'])
 
-        for f in formats:
-            f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
-            f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
         self._sort_formats(formats)
 
         display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id

From 1a1251e87716ba02bfe6c4c9af2726e413e00208 Mon Sep 17 00:00:00 2001
From: Mitsukarenai <mitsu@suumitsu.eu>
Date: Wed, 24 Jun 2015 10:40:01 +0200
Subject: [PATCH 0977/1515] [noco.tv] Fix issue #6066: title interpreted as
 integer

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 5bbd2dcf6..a53e27b27 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -195,7 +195,7 @@ class NocoIE(InfoExtractor):
         if episode_number:
             title += ' #' + compat_str(episode_number)
         if episode:
-            title += ' - ' + episode
+            title += ' - ' + compat_str(episode)
 
         description = show.get('show_resume') or show.get('family_resume')
 

From a219d175c66680968d5ce7de0a2e619623171a40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 20:48:57 +0600
Subject: [PATCH 0978/1515] [soundcloud] Improve _VALID_URL

Do not takeover soundcloud:user URLs
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c23c5ee0f..e31b4596a 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!sets/|likes/?(?:$|[?#]))
+                            (?!sets/|(?:likes|tracks)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)

From 03b9c94437756c3f715b29150daa59a8ea97eebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 20:49:22 +0600
Subject: [PATCH 0979/1515] [soundcloud:user] Add test

---
 youtube_dl/extractor/soundcloud.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e31b4596a..118ca4832 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -307,6 +307,9 @@ class SoundcloudUserIE(SoundcloudIE):
             'title': 'The Royal Concept',
         },
         'playlist_mincount': 1,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f843300fe56ffbfc8e3005fd0f7a8237e5deaaae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:12:13 +0600
Subject: [PATCH 0980/1515] [onionstudios] Add extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/onionstudios.py | 67 ++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/onionstudios.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc1a302e6..46cc4cd06 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -388,6 +388,7 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
+from .onionstudios import OnionStudiosIE
 from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
new file mode 100644
index 000000000..d5d03fd44
--- /dev/null
+++ b/youtube_dl/extractor/onionstudios.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import determine_ext
+
+
+class OnionStudiosIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:videos/[^/]+-|embed\?.*\bid=)(?P<id>\d+)(?!-)'
+
+    _TESTS = [{
+        'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
+        'md5': 'd4851405d31adfadf71cd7a487b765bb',
+        'info_dict': {
+            'id': '2937',
+            'ext': 'mp4',
+            'title': 'Hannibal charges forward, stops for a cocktail',
+            'description': 'md5:545299bda6abf87e5ec666548c6a9448',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'The A.V. Club',
+            'uploader_id': 'TheAVClub',
+        },
+    }, {
+        'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://www.onionstudios.com/embed?id=%s' % video_id, video_id)
+
+        formats = []
+        for src in re.findall(r'<source[^>]+src="([^"]+)"', webpage):
+            if determine_ext(src) != 'm3u8':  # m3u8 always results in 403
+                formats.append({
+                    'url': src,
+                })
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'share_title\s*=\s*"([^"]+)"', webpage, 'title')
+        description = self._search_regex(
+            r'share_description\s*=\s*"([^"]+)"', webpage,
+            'description', default=None)
+        thumbnail = self._search_regex(
+            r'poster="([^"]+)"', webpage, 'thumbnail', default=False)
+
+        uploader_id = self._search_regex(
+            r'twitter_handle\s*=\s*"([^"]+)"',
+            webpage, 'uploader id', fatal=False)
+        uploader = self._search_regex(
+            r'window\.channelName\s*=\s*"Embedded:([^"]+)"',
+            webpage, 'uploader', default=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
+        }

From d4f58034f7f3d959788d4fcf3812e24d3ba05f68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:16:33 +0600
Subject: [PATCH 0981/1515] [onionstudios] Add generic embed extraction routine

---
 youtube_dl/extractor/onionstudios.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index d5d03fd44..8fa507dec 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -27,6 +27,13 @@ class OnionStudiosIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/embed.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 1ac1c4c26ecdccb8c515aac634d965b89cbdabe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:19:50 +0600
Subject: [PATCH 0982/1515] [extractor/generic] Add support for OnionStudios
 embeds (Closes #5841)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7769ffc5c..9c5125f38 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -46,6 +46,7 @@ from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
+from .onionstudios import OnionStudiosIE
 
 
 class GenericIE(InfoExtractor):
@@ -1532,6 +1533,11 @@ class GenericIE(InfoExtractor):
         if dmcloud_url:
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
+        # Look for OnionStudios embeds
+        onionstudios_url = OnionStudiosIE._extract_url(webpage)
+        if onionstudios_url:
+            return self.url_result(onionstudios_url)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 8084be78c50e0e921508f13e69fda34a907dc5f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:23:16 +0600
Subject: [PATCH 0983/1515] [extractor/generic] Add test for OnionStudios
 embeds

---
 youtube_dl/extractor/generic.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9c5125f38..42e4e7035 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -837,6 +837,18 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpe?g$',
             }
         },
+        # OnionStudios embed
+        {
+            'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
+            'info_dict': {
+                'id': '2855',
+                'ext': 'mp4',
+                'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
+                'thumbnail': 're:^https?://.*\.jpe?g$',
+                'uploader': 'ClickHole',
+                'uploader_id': 'clickhole',
+            }
+        },
         # AdobeTVVideo embed
         {
             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',

From 12e9e8445dc090a12c6f7643a346725788b0ca6b Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Thu, 25 Jun 2015 01:10:42 +0600
Subject: [PATCH 0984/1515] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5f3a08f5a..10d2d4209 100644
--- a/README.md
+++ b/README.md
@@ -379,7 +379,7 @@ In February 2015, the new YouTube player contained a character sequence in a str
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
-These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--network-address` options](#network-options) to select another IP address.
+These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--source-address` options](#network-options) to select another IP address.
 
 ### SyntaxError: Non-ASCII character ###
 

From da92eeae42f556926cb676b3c14e270603b7e38e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Thu, 25 Jun 2015 00:31:32 -0500
Subject: [PATCH 0985/1515] Fix tests, description formatting

---
 youtube_dl/extractor/bbc.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index fed344ea0..bb671d473 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -428,6 +428,8 @@ class BBCNewsIE(BBCCoUkIE):
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
+            'upload_date': '20150324',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -438,8 +440,11 @@ class BBCNewsIE(BBCCoUkIE):
         'info_dict': {
             'id': 'NA',
             'ext': 'mp4',
-            'title': 'YPG - Tel Abyad..n tamam. kontrol.m.zde',
+            'title': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
+            'description': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
             'duration': 47,
+            'upload_date': '20150615',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -450,8 +455,11 @@ class BBCNewsIE(BBCCoUkIE):
         'info_dict': {
             'id': '39275083',
             'ext': 'mp4',
-            'title': 'Honduras militariza sus hospitales por nuevo esc.ndalo de corrupci.n',
+            'title': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
+            'description': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
             'duration': 87,
+            'upload_date': '20150619',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -507,7 +515,9 @@ class BBCNewsIE(BBCCoUkIE):
             title = jent.get('caption',list_title)
 
             duration = parse_duration(jent.get('duration'))
-            description = list_title + ' - ' + jent.get('caption','')
+            description = list_title
+            if jent.get('caption'):
+               description += ' - ' + jent.get('caption')
             thumbnail = None
             if jent.has_key('image'):
                thumbnail=jent['image'].get('href')
@@ -539,8 +549,12 @@ class BBCNewsIE(BBCCoUkIE):
                
             self._sort_formats(formats)
 
+            id = jent.get('id') if programme_id == None else programme_id
+            if id == None:
+               id = 'NA'
+
             ret.append( {
-                'id': jent.get('id') if programme_id == None else programme_id,
+                'id': id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From a1593a4a0e0f278d15ebda603df8c69679bb34c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Jun 2015 07:43:49 +0200
Subject: [PATCH 0986/1515] release 2015.06.25

---
 README.md              | 1 +
 docs/supportedsites.md | 7 ++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 10d2d4209..e3452c9e1 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ which means you can modify it, redistribute it or use it however you like.
     --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors
     --extractor-descriptions         Output descriptions of all supported extractors
+    --force-generic-extractor        Force extraction to use the generic extractor
     --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
                                      default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 220e52b98..9a50fbd1c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -17,6 +17,7 @@
  - **AcademicEarth:Course**
  - **AddAnime**
  - **AdobeTV**
+ - **AdobeTVVideo**
  - **AdultSwim**
  - **Aftenposten**
  - **Aftonbladet**
@@ -110,6 +111,7 @@
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
+ - **DailymotionCloud**
  - **daum.net**
  - **DBTV**
  - **DctpTv**
@@ -344,6 +346,7 @@
  - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
+ - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
  - **OpenFilm**
@@ -357,6 +360,7 @@
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
+ - **Pinkbike**
  - **Pladform**
  - **PlanetaPlay**
  - **play.fm**
@@ -612,6 +616,7 @@
  - **XBef**
  - **XboxClips**
  - **XHamster**
+ - **XHamsterEmbed**
  - **XMinus**
  - **XNXX**
  - **Xstream**
@@ -628,7 +633,7 @@
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
- - **Youku**
+ - **youku**
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34a13cb81..a225e03a1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.15'
+__version__ = '2015.06.25'

From 2182ab5187736f806f1a2ce959c3893179bdd1a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:15:13 +0800
Subject: [PATCH 0987/1515] [thesixtyone] Fix audio_server

Some of the songs are moved to Amazon AWS
---
 youtube_dl/extractor/thesixtyone.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index a77c6a2fc..e1be978a4 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -17,7 +17,7 @@ class TheSixtyOneIE(InfoExtractor):
             song
         )/(?P<id>[A-Za-z0-9]+)/?$'''
     _SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
-    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}.thesixtyone.com/thesixtyone_production/audio/{0:}_stream'
+    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}/thesixtyone_production/audio/{0:}_stream'
     _THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
     _TESTS = [
         {
@@ -78,6 +78,12 @@ class TheSixtyOneIE(InfoExtractor):
 
         song_data = json.loads(self._search_regex(
             r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+
+        if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
+            song_data['audio_server'] = 's3.amazonaws.com'
+        else:
+            song_data['audio_server'] = song_data['audio_server'] + '.thesixtyone.com'
+
         keys = [self._DECODE_MAP.get(s, s) for s in song_data['key']]
         url = self._SONG_FILE_URL_TEMPLATE.format(
             "".join(reversed(keys)), **song_data)

From 381c06775598da9667548fcc45d5df7a03748f7d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:19:04 +0800
Subject: [PATCH 0988/1515] [thesixtyone] Modernize

---
 youtube_dl/extractor/thesixtyone.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index e1be978a4..5d09eb9a8 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -1,9 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
@@ -70,14 +67,13 @@ class TheSixtyOneIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        song_id = mobj.group('id')
+        song_id = self._match_id(url)
 
         webpage = self._download_webpage(
             self._SONG_URL_TEMPLATE.format(song_id), song_id)
 
-        song_data = json.loads(self._search_regex(
-            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+        song_data = self._parse_json(self._search_regex(
+            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'), song_id)
 
         if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
             song_data['audio_server'] = 's3.amazonaws.com'

From b8091db6b9c75fbdc0ccaad3a9503ef091a7437c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:40:29 +0800
Subject: [PATCH 0989/1515] [planetaplay/quickvid/vube] Skip inaccessible sites

---
 youtube_dl/extractor/planetaplay.py | 3 ++-
 youtube_dl/extractor/quickvid.py    | 1 +
 youtube_dl/extractor/vube.py        | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/planetaplay.py b/youtube_dl/extractor/planetaplay.py
index 596c621d7..06505e96f 100644
--- a/youtube_dl/extractor/planetaplay.py
+++ b/youtube_dl/extractor/planetaplay.py
@@ -18,7 +18,8 @@ class PlanetaPlayIE(InfoExtractor):
             'id': '3586',
             'ext': 'flv',
             'title': 'md5:e829428ee28b1deed00de90de49d1da1',
-        }
+        },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     _SONG_FORMATS = {
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
index af7d76cf4..f414e2384 100644
--- a/youtube_dl/extractor/quickvid.py
+++ b/youtube_dl/extractor/quickvid.py
@@ -24,6 +24,7 @@ class QuickVidIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
             'view_count': int,
         },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 405cb9db4..149e36467 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -36,6 +36,7 @@ class VubeIE(InfoExtractor):
                 'comment_count': int,
                 'categories': ['amazing', 'hd', 'best drummer ever', 'william wei', 'bucket drumming', 'street drummer', 'epic street drumming'],
             },
+            'skip': 'Not accessible from Travis CI server',
         }, {
             'url': 'http://vube.com/Chiara+Grispo+Video+Channel/YL2qNPkqon',
             'md5': 'db7aba89d4603dadd627e9d1973946fe',

From 906e2f0eac3729228644fc5e9a3e8dedbd74894e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 25 Jun 2015 15:48:04 +0200
Subject: [PATCH 0990/1515] [downloader/external] Add downloader for httpie
 (closes #6079)

---
 youtube_dl/downloader/external.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7ca2d3143..a57c15856 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -109,6 +109,14 @@ class Aria2cFD(ExternalFD):
         cmd += ['--', info_dict['url']]
         return cmd
 
+
+class HttpieFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
+        for key, val in info_dict['http_headers'].items():
+            cmd += ['%s:%s' % (key, val)]
+        return cmd
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()

From 533f67d3fa76eb44799276656207a25ece6d20a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Jun 2015 19:54:44 +0600
Subject: [PATCH 0991/1515] [infoq] Relax _VALID_URL (Closes #6071)

---
 youtube_dl/extractor/infoq.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index f25f43664..117a7faf6 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -9,9 +9,9 @@ from ..compat import (
 
 
 class InfoQIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?infoq\.com/[^/]+/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?infoq\.com/(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things',
         'md5': 'b5ca0e0a8c1fed93b0e65e48e462f9a2',
         'info_dict': {
@@ -20,7 +20,10 @@ class InfoQIE(InfoExtractor):
             'description': 'Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.',
             'title': 'A Few of My Favorite [Python] Things',
         },
-    }
+    }, {
+        'url': 'http://www.infoq.com/fr/presentations/changez-avis-sur-javascript',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From b4dea075a3f923b4d85d74729479ec4c3e3a313a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:36:04 +0600
Subject: [PATCH 0992/1515] [lynda] Fix login request (Closes #6088)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index cfd3b14f4..2ff902c65 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -36,7 +36,7 @@ class LyndaBaseIE(InfoExtractor):
             'stayPut': 'false'
         }
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From 62cca96b729424fa71a3b174af7b646cc1cf12f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:46:42 +0600
Subject: [PATCH 0993/1515] [lynda] Fix confirm login request (#6088)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 2ff902c65..18efe5a09 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -65,7 +65,7 @@ class LyndaBaseIE(InfoExtractor):
                     'stayPut': 'false',
                 }
                 request = compat_urllib_request.Request(
-                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
+                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
                 login_page = self._download_webpage(
                     request, None,
                     'Confirming log in and log out from another device')

From 2988835af5cbe695e61d3b58bc65027f5b48fb63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:48:23 +0600
Subject: [PATCH 0994/1515] [lynda] Fix non-ASCII logins/passwords on python 2

---
 youtube_dl/extractor/lynda.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 18efe5a09..a00f6e5e5 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -30,8 +30,8 @@ class LyndaBaseIE(InfoExtractor):
             return
 
         login_form = {
-            'username': username,
-            'password': password,
+            'username': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
             'remember': 'false',
             'stayPut': 'false'
         }

From 2028c6e03d7e254831350081bb4b4741b0b47ac4 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Fri, 26 Jun 2015 21:27:43 +0530
Subject: [PATCH 0995/1515] Added a Playlist Info Extractor for WebOfStories

---
 youtube_dl/extractor/__init__.py     |  5 ++++-
 youtube_dl/extractor/webofstories.py | 25 +++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 46cc4cd06..c3f3a3e38 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -693,7 +693,10 @@ from .wdr import (
     WDRMobileIE,
     WDRMausIE,
 )
-from .webofstories import WebOfStoriesIE
+from .webofstories import (
+    WebOfStoriesIE,
+    WebOfStoriesPlaylistIE,
+)
 from .weibo import WeiboIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 73077a312..d70e30c00 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -98,3 +100,26 @@ class WebOfStoriesIE(InfoExtractor):
             'description': description,
             'duration': duration,
         }
+
+
+class WebOfStoriesPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
+    _TESTS = []
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.webofstories.com/play/%s' % video_number, 'WebOfStories')
+            for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
+        ]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Web\sof\sStories</title>', webpage, 'title')
+
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 03339b7b5bb8f563eace8826512e2f7a4baba415 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 18:25:43 +0100
Subject: [PATCH 0996/1515] [snagfilms] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/snagfilms.py | 67 +++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/snagfilms.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc1a302e6..3b906b880 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -492,6 +492,7 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
+from .snagfilms import SnagFilmsIE
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soompi import (
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
new file mode 100644
index 000000000..33832855f
--- /dev/null
+++ b/youtube_dl/extractor/snagfilms.py
@@ -0,0 +1,67 @@
+from .common import InfoExtractor
+from ..utils import js_to_json
+from re import DOTALL
+
+class SnagFilmsIE(InfoExtractor):
+	_VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
+	_TEST = {
+		'url': 'http://www.snagfilms.com/films/title/lost_for_life',
+		'info_dict':
+		{
+			'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
+			'display_id': 'lost_for_life',
+			'ext': 'mp4',
+			'title': 'Lost for Life',
+			'duration': 4489,
+			'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
+			'categories': ['Documentary','Crime','Award Winning','Festivals']
+		}
+	}
+
+	def _real_extract(self, url):
+		display_id = self._search_regex(
+			self._VALID_URL,
+			url,
+			'display_id',
+			group='display_id'
+		)
+		webpage = self._download_webpage(url, display_id)
+
+		json_data = self._parse_json(self._html_search_regex(
+			r'"data":{"film":(?P<data>{.*?}})}',
+			webpage,
+			'data',
+			group='data'
+		), display_id)
+		title = json_data['title']
+		video_id = json_data['id']
+		duration = int(json_data['duration'])
+		description = json_data['synopsis']
+		categories = [category['title'] for category in json_data['categories']]
+
+		embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+		sources = self._parse_json(js_to_json(self._html_search_regex(
+			r'sources: (?P<sources>\[.*?\])',
+			embed_webpage,
+			'sources',
+			group='sources',
+			flags=DOTALL
+		)), video_id)
+
+		formats = []
+		for source in sources:
+			if source['type'] == 'm3u8':
+				formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+			else:
+				formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
+		self._sort_formats(formats)
+
+		return {
+			'id': video_id,
+			'display_id': display_id,
+			'title': title,
+			'duration': duration,
+			'description': description,
+			'categories': categories,
+			'formats': formats,
+		}

From d8d24a922adb0646f9f78b2fc4a287a72fa7ff73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 00:36:23 +0600
Subject: [PATCH 0997/1515] [youtube] Extract formats from multiple DASH
 manifests (Closes #6093)

DASH manifest pointed by dashmpd from the video webpage and one pointed by get_video_info may
be different (namely different itag set) - some itags are missing from DASH manifest pointed by
webpage's dashmpd, some - from DASH manifest pointed by get_video_info's dashmpd).
The general idea is to take a union of itags of both DASH manifests (for example video with such
'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093).
---
 youtube_dl/extractor/youtube.py | 77 ++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..fa1a2e544 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -853,6 +853,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             player_url = None
 
+        dash_mpds = []
+
+        def add_dash_mpd(video_info):
+            dash_mpd = video_info.get('dashmpd')
+            if dash_mpd and dash_mpd[0] not in dash_mpds:
+                dash_mpds.append(dash_mpd[0])
+
         # Get video info
         embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
@@ -873,34 +880,40 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 note='Refetching age-gated info webpage',
                 errnote='unable to download video info webpage')
             video_info = compat_parse_qs(video_info_webpage)
+            add_dash_mpd(video_info)
         else:
             age_gate = False
-            try:
-                # Try looking directly into the video webpage
-                mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
-                if not mobj:
-                    raise ValueError('Could not find ytplayer.config')  # caught below
+            # Try looking directly into the video webpage
+            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+            if mobj:
                 json_code = uppercase_escape(mobj.group(1))
                 ytplayer_config = json.loads(json_code)
                 args = ytplayer_config['args']
-                # Convert to the same format returned by compat_parse_qs
-                video_info = dict((k, [v]) for k, v in args.items())
-                if not args.get('url_encoded_fmt_stream_map'):
-                    raise ValueError('No stream_map present')  # caught below
-            except ValueError:
-                # We fallback to the get_video_info pages (used by the embed page)
-                self.report_video_info_webpage_download(video_id)
-                for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (proto, video_id, el_type))
-                    video_info_webpage = self._download_webpage(
-                        video_info_url,
-                        video_id, note=False,
-                        errnote='unable to download video info webpage')
-                    video_info = compat_parse_qs(video_info_webpage)
-                    if 'token' in video_info:
-                        break
+                if args.get('url_encoded_fmt_stream_map'):
+                    # Convert to the same format returned by compat_parse_qs
+                    video_info = dict((k, [v]) for k, v in args.items())
+                    add_dash_mpd(video_info)
+            # We also try looking in get_video_info since it may contain different dashmpd
+            # URL that points to a DASH manifest with possibly different itag set (some itags
+            # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
+            # manifest pointed by get_video_info's dashmpd).
+            # The general idea is to take a union of itags of both DASH manifests (for example
+            # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+            self.report_video_info_webpage_download(video_id)
+            for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                video_info_url = (
+                    '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                    % (proto, video_id, el_type))
+                video_info_webpage = self._download_webpage(
+                    video_info_url,
+                    video_id, note=False,
+                    errnote='unable to download video info webpage')
+                get_video_info = compat_parse_qs(video_info_webpage)
+                add_dash_mpd(get_video_info)
+                if not video_info:
+                    video_info = get_video_info
+                if 'token' in get_video_info:
+                    break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(
@@ -1118,24 +1131,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
-            dash_mpd = video_info.get('dashmpd')
-            if dash_mpd:
-                dash_manifest_url = dash_mpd[0]
+            for dash_manifest_url in dash_mpds:
+                dash_formats = {}
                 try:
-                    dash_formats = self._parse_dash_manifest(
-                        video_id, dash_manifest_url, player_url, age_gate)
+                    for df in self._parse_dash_manifest(
+                            video_id, dash_manifest_url, player_url, age_gate):
+                        # Do not overwrite DASH format found in some previous DASH manifest
+                        if df['format_id'] not in dash_formats:
+                            dash_formats[df['format_id']] = df
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
-                else:
+                if dash_formats:
                     # Remove the formats we found through non-DASH, they
                     # contain less info and it can be wrong, because we use
                     # fixed values (for example the resolution). See
                     # https://github.com/rg3/youtube-dl/issues/5774 for an
                     # example.
-                    dash_keys = set(df['format_id'] for df in dash_formats)
+                    dash_keys = set(df['format_id'] for df in dash_formats.values())
                     formats = [f for f in formats if f['format_id'] not in dash_keys]
-                    formats.extend(dash_formats)
+                    formats.extend(dash_formats.values())
 
         # Check for malformed aspect ratio
         stretched_m = re.search(

From 1b5a1ae257c6d64fd69ea91eaf69328d4b6d17df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 00:41:26 +0600
Subject: [PATCH 0998/1515] [youtube] Pick up codecs info from DASH manifest
 when not set explicitly

---
 youtube_dl/extractor/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..c9d8e5125 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -824,6 +824,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     except StopIteration:
                         full_info = self._formats.get(format_id, {}).copy()
                         full_info.update(f)
+                        codecs = r.attrib.get('codecs')
+                        if codecs:
+                            if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
+                                full_info['vcodec'] = codecs
+                            elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
+                                full_info['acodec'] = codecs
                         formats.append(full_info)
                     else:
                         existing_format.update(f)

From d80265ccd6ff08ef273d83aff847a457f051071f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 02:48:50 +0600
Subject: [PATCH 0999/1515] [youtube] Simplify non-DASH formats exclusion

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fa1a2e544..46841617a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1148,8 +1148,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # fixed values (for example the resolution). See
                     # https://github.com/rg3/youtube-dl/issues/5774 for an
                     # example.
-                    dash_keys = set(df['format_id'] for df in dash_formats.values())
-                    formats = [f for f in formats if f['format_id'] not in dash_keys]
+                    formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
                     formats.extend(dash_formats.values())
 
         # Check for malformed aspect ratio

From 7e0480ae0e7a9d0b3c73366b8d9ecf86226ad5a8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 21:50:27 +0100
Subject: [PATCH 1000/1515] convert tabs to 4 spaces identation

---
 youtube_dl/extractor/snagfilms.py | 114 +++++++++++++++---------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 33832855f..212fefef7 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -3,65 +3,65 @@ from ..utils import js_to_json
 from re import DOTALL
 
 class SnagFilmsIE(InfoExtractor):
-	_VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
-	_TEST = {
-		'url': 'http://www.snagfilms.com/films/title/lost_for_life',
-		'info_dict':
-		{
-			'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
-			'display_id': 'lost_for_life',
-			'ext': 'mp4',
-			'title': 'Lost for Life',
-			'duration': 4489,
-			'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
-			'categories': ['Documentary','Crime','Award Winning','Festivals']
-		}
-	}
+    _VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
+    _TEST = {
+        'url': 'http://www.snagfilms.com/films/title/lost_for_life',
+        'info_dict':
+        {
+            'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
+            'display_id': 'lost_for_life',
+            'ext': 'mp4',
+            'title': 'Lost for Life',
+            'duration': 4489,
+            'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
+            'categories': ['Documentary','Crime','Award Winning','Festivals']
+        }
+    }
 
-	def _real_extract(self, url):
-		display_id = self._search_regex(
-			self._VALID_URL,
-			url,
-			'display_id',
-			group='display_id'
-		)
-		webpage = self._download_webpage(url, display_id)
+    def _real_extract(self, url):
+        display_id = self._search_regex(
+            self._VALID_URL,
+            url,
+            'display_id',
+            group='display_id'
+        )
+        webpage = self._download_webpage(url, display_id)
 
-		json_data = self._parse_json(self._html_search_regex(
-			r'"data":{"film":(?P<data>{.*?}})}',
-			webpage,
-			'data',
-			group='data'
-		), display_id)
-		title = json_data['title']
-		video_id = json_data['id']
-		duration = int(json_data['duration'])
-		description = json_data['synopsis']
-		categories = [category['title'] for category in json_data['categories']]
+        json_data = self._parse_json(self._html_search_regex(
+            r'"data":{"film":(?P<data>{.*?}})}',
+            webpage,
+            'data',
+            group='data'
+        ), display_id)
+        title = json_data['title']
+        video_id = json_data['id']
+        duration = int(json_data['duration'])
+        description = json_data['synopsis']
+        categories = [category['title'] for category in json_data['categories']]
 
-		embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-		sources = self._parse_json(js_to_json(self._html_search_regex(
-			r'sources: (?P<sources>\[.*?\])',
-			embed_webpage,
-			'sources',
-			group='sources',
-			flags=DOTALL
-		)), video_id)
+        embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+        sources = self._parse_json(js_to_json(self._html_search_regex(
+            r'sources: (?P<sources>\[.*?\])',
+            embed_webpage,
+            'sources',
+            group='sources',
+            flags=DOTALL
+        )), video_id)
 
-		formats = []
-		for source in sources:
-			if source['type'] == 'm3u8':
-				formats.extend(self._extract_m3u8_formats(source['file'], video_id))
-			else:
-				formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
-		self._sort_formats(formats)
+        formats = []
+        for source in sources:
+            if source['type'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+            else:
+                formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
+        self._sort_formats(formats)
 
-		return {
-			'id': video_id,
-			'display_id': display_id,
-			'title': title,
-			'duration': duration,
-			'description': description,
-			'categories': categories,
-			'formats': formats,
-		}
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'duration': duration,
+            'description': description,
+            'categories': categories,
+            'formats': formats,
+        }

From fd40bdc0be8984bf6043e70796c1d465a0499d03 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 21:56:15 +0100
Subject: [PATCH 1001/1515] remove unnecessary symbolic name for group

---
 youtube_dl/extractor/snagfilms.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 212fefef7..60354b9af 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -31,7 +31,6 @@ class SnagFilmsIE(InfoExtractor):
             r'"data":{"film":(?P<data>{.*?}})}',
             webpage,
             'data',
-            group='data'
         ), display_id)
         title = json_data['title']
         video_id = json_data['id']
@@ -44,7 +43,6 @@ class SnagFilmsIE(InfoExtractor):
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,
             'sources',
-            group='sources',
             flags=DOTALL
         )), video_id)
 

From 7d7d4690259f343385a240efcef8d157fc99c72d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 27 Jun 2015 00:13:14 +0100
Subject: [PATCH 1002/1515] add support for embed links

---
 youtube_dl/extractor/snagfilms.py | 48 ++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 60354b9af..cd345474e 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -1,10 +1,10 @@
+from re import match,DOTALL
 from .common import InfoExtractor
 from ..utils import js_to_json
-from re import DOTALL
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
-    _TEST = {
+    _VALID_URL = r'(?:https?://)?(?:www.|embed.)?snagfilms\.com/(?:films/title/(?P<display_id>.+?)|embed/player\?.*filmId=(?P<id>.+?))(?:&|/|$)'
+    _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'info_dict':
         {
@@ -16,29 +16,48 @@ class SnagFilmsIE(InfoExtractor):
             'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
             'categories': ['Documentary','Crime','Award Winning','Festivals']
         }
-    }
+    },{
+        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831',
+        'info_dict':
+        {
+            'id': '74849a00-85a9-11e1-9660-123139220831',
+            'display_id': 'while_we_watch',
+            'ext': 'mp4',
+            'title': '#whilewewatch',
+            'duration': 2311,
+            'description': 'A gripping portrait of the Occupy Wall Street media revolution,&nbsp;#WHILEWEWATCH is the first definitive film to emerge from Zuccotti Park—with full access and cooperation from masterminds who made #OccupyWallStreet a reality.&nbsp;The #OccupyWallStreet media team had no fear of a critical city government, big corporations, hostile police or a lagging mainstream media to tell their story. Through rain, snow, grueling days and sleeping on concrete, they pump out exhilarating ideas to the world. With little money, they rely on Twitter, texting, Wi-Fi, posters, Tumblr, live streams, YouTube, Facebook, dramatic marches, drumbeats and chants. As the film unfolds, we witness the burgeoning power of social media.<br />',
+            'categories': ['Documentary','Politics']
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id = self._search_regex(
-            self._VALID_URL,
-            url,
-            'display_id',
-            group='display_id'
-        )
-        webpage = self._download_webpage(url, display_id)
+        display_id, video_id = match(self._VALID_URL,url).groups()
+        if display_id is None:
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+
+            display_id = self._html_search_regex(
+                r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
+                embed_webpage,
+                'display_id'
+            )
+        webpage = self._download_webpage('http://www.snagfilms.com/films/title/' + display_id, display_id)
 
         json_data = self._parse_json(self._html_search_regex(
             r'"data":{"film":(?P<data>{.*?}})}',
             webpage,
-            'data',
+            'data'
         ), display_id)
+
         title = json_data['title']
-        video_id = json_data['id']
         duration = int(json_data['duration'])
         description = json_data['synopsis']
         categories = [category['title'] for category in json_data['categories']]
+        thumbnail = json_data['image']
+
+        if video_id is None:
+            video_id = json_data['id']
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
 
-        embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
         sources = self._parse_json(js_to_json(self._html_search_regex(
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,
@@ -61,5 +80,6 @@ class SnagFilmsIE(InfoExtractor):
             'duration': duration,
             'description': description,
             'categories': categories,
+            'thumbnail': thumbnail,
             'formats': formats,
         }

From 4eb10f66216b5239a9a8efcf950c9b16f5c8b5de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Jun 2015 13:13:57 +0800
Subject: [PATCH 1003/1515] [utils] Add ISO3166Utils

---
 youtube_dl/utils.py | 260 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 260 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2746b2d1..96490f112 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2084,6 +2084,266 @@ class ISO639Utils(object):
                 return short_name
 
 
+class ISO3166Utils(object):
+    # From http://data.okfn.org/data/core/country-list
+    _country_map = {
+        'AF': 'Afghanistan',
+        'AX': 'Åland Islands',
+        'AL': 'Albania',
+        'DZ': 'Algeria',
+        'AS': 'American Samoa',
+        'AD': 'Andorra',
+        'AO': 'Angola',
+        'AI': 'Anguilla',
+        'AQ': 'Antarctica',
+        'AG': 'Antigua and Barbuda',
+        'AR': 'Argentina',
+        'AM': 'Armenia',
+        'AW': 'Aruba',
+        'AU': 'Australia',
+        'AT': 'Austria',
+        'AZ': 'Azerbaijan',
+        'BS': 'Bahamas',
+        'BH': 'Bahrain',
+        'BD': 'Bangladesh',
+        'BB': 'Barbados',
+        'BY': 'Belarus',
+        'BE': 'Belgium',
+        'BZ': 'Belize',
+        'BJ': 'Benin',
+        'BM': 'Bermuda',
+        'BT': 'Bhutan',
+        'BO': 'Bolivia, Plurinational State of',
+        'BQ': 'Bonaire, Sint Eustatius and Saba',
+        'BA': 'Bosnia and Herzegovina',
+        'BW': 'Botswana',
+        'BV': 'Bouvet Island',
+        'BR': 'Brazil',
+        'IO': 'British Indian Ocean Territory',
+        'BN': 'Brunei Darussalam',
+        'BG': 'Bulgaria',
+        'BF': 'Burkina Faso',
+        'BI': 'Burundi',
+        'KH': 'Cambodia',
+        'CM': 'Cameroon',
+        'CA': 'Canada',
+        'CV': 'Cape Verde',
+        'KY': 'Cayman Islands',
+        'CF': 'Central African Republic',
+        'TD': 'Chad',
+        'CL': 'Chile',
+        'CN': 'China',
+        'CX': 'Christmas Island',
+        'CC': 'Cocos (Keeling) Islands',
+        'CO': 'Colombia',
+        'KM': 'Comoros',
+        'CG': 'Congo',
+        'CD': 'Congo, the Democratic Republic of the',
+        'CK': 'Cook Islands',
+        'CR': 'Costa Rica',
+        'CI': 'Côte d\'Ivoire',
+        'HR': 'Croatia',
+        'CU': 'Cuba',
+        'CW': 'Curaçao',
+        'CY': 'Cyprus',
+        'CZ': 'Czech Republic',
+        'DK': 'Denmark',
+        'DJ': 'Djibouti',
+        'DM': 'Dominica',
+        'DO': 'Dominican Republic',
+        'EC': 'Ecuador',
+        'EG': 'Egypt',
+        'SV': 'El Salvador',
+        'GQ': 'Equatorial Guinea',
+        'ER': 'Eritrea',
+        'EE': 'Estonia',
+        'ET': 'Ethiopia',
+        'FK': 'Falkland Islands (Malvinas)',
+        'FO': 'Faroe Islands',
+        'FJ': 'Fiji',
+        'FI': 'Finland',
+        'FR': 'France',
+        'GF': 'French Guiana',
+        'PF': 'French Polynesia',
+        'TF': 'French Southern Territories',
+        'GA': 'Gabon',
+        'GM': 'Gambia',
+        'GE': 'Georgia',
+        'DE': 'Germany',
+        'GH': 'Ghana',
+        'GI': 'Gibraltar',
+        'GR': 'Greece',
+        'GL': 'Greenland',
+        'GD': 'Grenada',
+        'GP': 'Guadeloupe',
+        'GU': 'Guam',
+        'GT': 'Guatemala',
+        'GG': 'Guernsey',
+        'GN': 'Guinea',
+        'GW': 'Guinea-Bissau',
+        'GY': 'Guyana',
+        'HT': 'Haiti',
+        'HM': 'Heard Island and McDonald Islands',
+        'VA': 'Holy See (Vatican City State)',
+        'HN': 'Honduras',
+        'HK': 'Hong Kong',
+        'HU': 'Hungary',
+        'IS': 'Iceland',
+        'IN': 'India',
+        'ID': 'Indonesia',
+        'IR': 'Iran, Islamic Republic of',
+        'IQ': 'Iraq',
+        'IE': 'Ireland',
+        'IM': 'Isle of Man',
+        'IL': 'Israel',
+        'IT': 'Italy',
+        'JM': 'Jamaica',
+        'JP': 'Japan',
+        'JE': 'Jersey',
+        'JO': 'Jordan',
+        'KZ': 'Kazakhstan',
+        'KE': 'Kenya',
+        'KI': 'Kiribati',
+        'KP': 'Korea, Democratic People\'s Republic of',
+        'KR': 'Korea, Republic of',
+        'KW': 'Kuwait',
+        'KG': 'Kyrgyzstan',
+        'LA': 'Lao People\'s Democratic Republic',
+        'LV': 'Latvia',
+        'LB': 'Lebanon',
+        'LS': 'Lesotho',
+        'LR': 'Liberia',
+        'LY': 'Libya',
+        'LI': 'Liechtenstein',
+        'LT': 'Lithuania',
+        'LU': 'Luxembourg',
+        'MO': 'Macao',
+        'MK': 'Macedonia, the Former Yugoslav Republic of',
+        'MG': 'Madagascar',
+        'MW': 'Malawi',
+        'MY': 'Malaysia',
+        'MV': 'Maldives',
+        'ML': 'Mali',
+        'MT': 'Malta',
+        'MH': 'Marshall Islands',
+        'MQ': 'Martinique',
+        'MR': 'Mauritania',
+        'MU': 'Mauritius',
+        'YT': 'Mayotte',
+        'MX': 'Mexico',
+        'FM': 'Micronesia, Federated States of',
+        'MD': 'Moldova, Republic of',
+        'MC': 'Monaco',
+        'MN': 'Mongolia',
+        'ME': 'Montenegro',
+        'MS': 'Montserrat',
+        'MA': 'Morocco',
+        'MZ': 'Mozambique',
+        'MM': 'Myanmar',
+        'NA': 'Namibia',
+        'NR': 'Nauru',
+        'NP': 'Nepal',
+        'NL': 'Netherlands',
+        'NC': 'New Caledonia',
+        'NZ': 'New Zealand',
+        'NI': 'Nicaragua',
+        'NE': 'Niger',
+        'NG': 'Nigeria',
+        'NU': 'Niue',
+        'NF': 'Norfolk Island',
+        'MP': 'Northern Mariana Islands',
+        'NO': 'Norway',
+        'OM': 'Oman',
+        'PK': 'Pakistan',
+        'PW': 'Palau',
+        'PS': 'Palestine, State of',
+        'PA': 'Panama',
+        'PG': 'Papua New Guinea',
+        'PY': 'Paraguay',
+        'PE': 'Peru',
+        'PH': 'Philippines',
+        'PN': 'Pitcairn',
+        'PL': 'Poland',
+        'PT': 'Portugal',
+        'PR': 'Puerto Rico',
+        'QA': 'Qatar',
+        'RE': 'Réunion',
+        'RO': 'Romania',
+        'RU': 'Russian Federation',
+        'RW': 'Rwanda',
+        'BL': 'Saint Barthélemy',
+        'SH': 'Saint Helena, Ascension and Tristan da Cunha',
+        'KN': 'Saint Kitts and Nevis',
+        'LC': 'Saint Lucia',
+        'MF': 'Saint Martin (French part)',
+        'PM': 'Saint Pierre and Miquelon',
+        'VC': 'Saint Vincent and the Grenadines',
+        'WS': 'Samoa',
+        'SM': 'San Marino',
+        'ST': 'Sao Tome and Principe',
+        'SA': 'Saudi Arabia',
+        'SN': 'Senegal',
+        'RS': 'Serbia',
+        'SC': 'Seychelles',
+        'SL': 'Sierra Leone',
+        'SG': 'Singapore',
+        'SX': 'Sint Maarten (Dutch part)',
+        'SK': 'Slovakia',
+        'SI': 'Slovenia',
+        'SB': 'Solomon Islands',
+        'SO': 'Somalia',
+        'ZA': 'South Africa',
+        'GS': 'South Georgia and the South Sandwich Islands',
+        'SS': 'South Sudan',
+        'ES': 'Spain',
+        'LK': 'Sri Lanka',
+        'SD': 'Sudan',
+        'SR': 'Suriname',
+        'SJ': 'Svalbard and Jan Mayen',
+        'SZ': 'Swaziland',
+        'SE': 'Sweden',
+        'CH': 'Switzerland',
+        'SY': 'Syrian Arab Republic',
+        'TW': 'Taiwan, Province of China',
+        'TJ': 'Tajikistan',
+        'TZ': 'Tanzania, United Republic of',
+        'TH': 'Thailand',
+        'TL': 'Timor-Leste',
+        'TG': 'Togo',
+        'TK': 'Tokelau',
+        'TO': 'Tonga',
+        'TT': 'Trinidad and Tobago',
+        'TN': 'Tunisia',
+        'TR': 'Turkey',
+        'TM': 'Turkmenistan',
+        'TC': 'Turks and Caicos Islands',
+        'TV': 'Tuvalu',
+        'UG': 'Uganda',
+        'UA': 'Ukraine',
+        'AE': 'United Arab Emirates',
+        'GB': 'United Kingdom',
+        'US': 'United States',
+        'UM': 'United States Minor Outlying Islands',
+        'UY': 'Uruguay',
+        'UZ': 'Uzbekistan',
+        'VU': 'Vanuatu',
+        'VE': 'Venezuela, Bolivarian Republic of',
+        'VN': 'Viet Nam',
+        'VG': 'Virgin Islands, British',
+        'VI': 'Virgin Islands, U.S.',
+        'WF': 'Wallis and Futuna',
+        'EH': 'Western Sahara',
+        'YE': 'Yemen',
+        'ZM': 'Zambia',
+        'ZW': 'Zimbabwe',
+    }
+
+    @classmethod
+    def short2full(cls, code):
+        """Convert an ISO 3166-2 country code to the corresponding full name"""
+        return cls._country_map.get(code.upper())
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From af214c3a79bb598d3a13559c5ebddcca4584669a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Jun 2015 13:15:57 +0800
Subject: [PATCH 1004/1515] [youtube] More useful messages for georestricted
 videos (#5716)

---
 youtube_dl/extractor/youtube.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..d9240ff02 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -32,6 +32,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     uppercase_escape,
+    ISO3166Utils,
 )
 
 
@@ -903,6 +904,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         break
         if 'token' not in video_info:
             if 'reason' in video_info:
+                if 'The uploader has not made this video available in your country.' in video_info['reason']:
+                    regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
+                    if regions_allowed is not None:
+                        raise ExtractorError('YouTube said: This video is available in %s only' % (
+                            ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
+                            expected=True)
                 raise ExtractorError(
                     'YouTube said: %s' % video_info['reason'][0],
                     expected=True, video_id=video_id)

From bc93bdb5bbf48b121977a5088ab9607f0fbeb83e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 13:19:46 +0600
Subject: [PATCH 1005/1515] [youtube] Fix reference before assignment for
 video_info

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 46841617a..45e5cb80e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -883,6 +883,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             add_dash_mpd(video_info)
         else:
             age_gate = False
+            video_info = None
             # Try looking directly into the video webpage
             mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
             if mobj:

From 226b886ca85b5843d2cb9638fb6d88223e9dd5b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:04:55 +0600
Subject: [PATCH 1006/1515] [vk] Fix authentication (Closes #6105)

---
 youtube_dl/extractor/vk.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 38ff3c1a9..f2ae109f9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -121,20 +121,27 @@ class VKIE(InfoExtractor):
         if username is None:
             return
 
-        login_form = {
-            'act': 'login',
-            'role': 'al_frame',
-            'expire': '1',
+        login_page = self._download_webpage(
+            'https://vk.com', None, 'Downloading login page')
+
+        login_form = dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            login_page))
+
+        login_form.update({
             'email': username.encode('cp1251'),
             'pass': password.encode('cp1251'),
-        }
+        })
 
-        request = compat_urllib_request.Request('https://login.vk.com/?act=login',
-                                                compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+        request = compat_urllib_request.Request(
+            'https://login.vk.com/?act=login',
+            compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        login_page = self._download_webpage(
+            request, None, note='Logging in as %s' % username)
 
         if re.search(r'onLoginFailed', login_page):
-            raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
+            raise ExtractorError(
+                'Unable to login, incorrect username and/or password', expected=True)
 
     def _real_initialize(self):
         self._login()

From 00334d0de01d46165880569159c534f9b7d927a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:26:51 +0600
Subject: [PATCH 1007/1515] [options] Add missing whitespace and split lines

---
 youtube_dl/options.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6aeca61ee..e7d067642 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -351,7 +351,8 @@ def parseOpts(overrideArguments=None):
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,
         help=(
-            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.'
+            'If a merge is required (e.g. bestvideo+bestaudio), '
+            'output to given container format. One of mkv, mp4, ogg, webm, flv. '
             'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')

From 0a3cf9ad3dc1a038018ff347473c0e2d8fc10a69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:31:18 +0600
Subject: [PATCH 1008/1515] [youtube] Skip get_video_info requests when
 --youtube-skip-dash-manifest is specified

---
 youtube_dl/extractor/youtube.py | 43 +++++++++++++++++----------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 45e5cb80e..2c9ad5e92 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -894,27 +894,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
-            # We also try looking in get_video_info since it may contain different dashmpd
-            # URL that points to a DASH manifest with possibly different itag set (some itags
-            # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
-            # manifest pointed by get_video_info's dashmpd).
-            # The general idea is to take a union of itags of both DASH manifests (for example
-            # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
-            self.report_video_info_webpage_download(video_id)
-            for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                video_info_url = (
-                    '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                    % (proto, video_id, el_type))
-                video_info_webpage = self._download_webpage(
-                    video_info_url,
-                    video_id, note=False,
-                    errnote='unable to download video info webpage')
-                get_video_info = compat_parse_qs(video_info_webpage)
-                add_dash_mpd(get_video_info)
-                if not video_info:
-                    video_info = get_video_info
-                if 'token' in get_video_info:
-                    break
+            if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
+                # We also try looking in get_video_info since it may contain different dashmpd
+                # URL that points to a DASH manifest with possibly different itag set (some itags
+                # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
+                # manifest pointed by get_video_info's dashmpd).
+                # The general idea is to take a union of itags of both DASH manifests (for example
+                # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+                self.report_video_info_webpage_download(video_id)
+                for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                    video_info_url = (
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (proto, video_id, el_type))
+                    video_info_webpage = self._download_webpage(
+                        video_info_url,
+                        video_id, note=False,
+                        errnote='unable to download video info webpage')
+                    get_video_info = compat_parse_qs(video_info_webpage)
+                    add_dash_mpd(get_video_info)
+                    if not video_info:
+                        video_info = get_video_info
+                    if 'token' in get_video_info:
+                        break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(

From b2575b38e76e14b7552a43cd707a03fe78748f40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:38:41 +0600
Subject: [PATCH 1009/1515] [options] Clarify --youtube-skip-dash-manifest

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6aeca61ee..e3dfb7af9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -346,7 +346,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--youtube-skip-dash-manifest',
         action='store_false', dest='youtube_include_dash_manifest',
-        help='Do not download the DASH manifest on YouTube videos')
+        help='Do not download the DASH manifests and related data on YouTube videos')
     video_format.add_option(
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,

From da77d856a1310b52975abbad82121a1b4c6597a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:55:46 +0600
Subject: [PATCH 1010/1515] [youtube] Add test for #6093

---
 youtube_dl/extractor/youtube.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2c9ad5e92..20e1781f8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -518,6 +518,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
+        {
+            'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
+            'info_dict': {
+                'id': 'FIl7x6_3R5Y',
+                'ext': 'mp4',
+                'title': 'md5:7b81415841e02ecd4313668cde88737a',
+                'description': 'md5:116377fd2963b81ec4ce64b542173306',
+                'upload_date': '20150625',
+                'uploader_id': 'dorappi2000',
+                'uploader': 'dorappi2000',
+                'formats': 'mincount:33',
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From f39eb98bab497d8e06b9f243a8240509326678f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 27 Jun 2015 10:55:25 +0100
Subject: [PATCH 1011/1515] download all pages before start extracting info

---
 youtube_dl/extractor/snagfilms.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cd345474e..74cd2698d 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -34,7 +34,6 @@ class SnagFilmsIE(InfoExtractor):
         display_id, video_id = match(self._VALID_URL,url).groups()
         if display_id is None:
             embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-
             display_id = self._html_search_regex(
                 r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
                 embed_webpage,
@@ -48,16 +47,16 @@ class SnagFilmsIE(InfoExtractor):
             'data'
         ), display_id)
 
+        if video_id is None:
+            video_id = json_data['id']
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+
         title = json_data['title']
         duration = int(json_data['duration'])
         description = json_data['synopsis']
         categories = [category['title'] for category in json_data['categories']]
         thumbnail = json_data['image']
 
-        if video_id is None:
-            video_id = json_data['id']
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-
         sources = self._parse_json(js_to_json(self._html_search_regex(
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,

From 73c471e9efd3f888af060b391eb750b079a5822c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Jun 2015 14:18:01 +0200
Subject: [PATCH 1012/1515] [newstube] Fix GUID extraction (fixes #6109)

---
 youtube_dl/extractor/newstube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index 85fcad06b..fe29cfc3f 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -31,7 +31,7 @@ class NewstubeIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         video_guid = self._html_search_regex(
-            r'<meta property="og:video" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+                r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             page, 'video GUID')
 
         player = self._download_xml(

From cee16e0fa3c2030aa6e58669ca2253457f1f3e64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Jun 2015 14:20:33 +0200
Subject: [PATCH 1013/1515] [newstube] style: fix alignment

---
 youtube_dl/extractor/newstube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index fe29cfc3f..5a9e73cd6 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -31,7 +31,7 @@ class NewstubeIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         video_guid = self._html_search_regex(
-                r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             page, 'video GUID')
 
         player = self._download_xml(

From 654fd03c73fa0e4407a71c07d821b45321c3cdb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:20:42 +0600
Subject: [PATCH 1014/1515] [snagfilms] Improve and simplify

---
 youtube_dl/extractor/__init__.py  |   5 +-
 youtube_dl/extractor/snagfilms.py | 171 +++++++++++++++++++-----------
 2 files changed, 114 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 59068a8b8..7e74a971d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -493,7 +493,10 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
-from .snagfilms import SnagFilmsIE
+from .snagfilms import (
+    SnagFilmsIE,
+    SnagFilmsEmbedIE,
+)
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soompi import (
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 74cd2698d..6e103bd49 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -1,84 +1,133 @@
-from re import match,DOTALL
+from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    parse_duration,
+)
+
+
+class SnagFilmsEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|embed)\.)?snagfilms\.com/embed/player\?.*\bfilmId=(?P<id>[\da-f-]{36})'
+    _TESTS = [{
+        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
+        'md5': '2924e9215c6eff7a55ed35b72276bd93',
+        'info_dict': {
+            'id': '74849a00-85a9-11e1-9660-123139220831',
+            'ext': 'mp4',
+            'title': '#whilewewatch',
+        }
+    }, {
+        'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for source in self._parse_json(js_to_json(self._search_regex(
+                r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):
+            file_ = source.get('file')
+            if not file_:
+                continue
+            type_ = source.get('type')
+            format_id = source.get('label')
+            ext = determine_ext(file_)
+            if any(_ == 'm3u8' for _ in (type_, ext)):
+                formats.extend(self._extract_m3u8_formats(
+                    file_, video_id, 'mp4', m3u8_id='hls'))
+            else:
+                bitrate = int_or_none(self._search_regex(
+                    r'(\d+)kbps', file_, 'bitrate', default=None))
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]$', format_id, 'height', default=None))
+                formats.append({
+                    'url': file_,
+                    'format_id': format_id,
+                    'tbr': bitrate,
+                    'height': height,
+                })
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
+            webpage, 'title')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }
+
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www.|embed.)?snagfilms\.com/(?:films/title/(?P<display_id>.+?)|embed/player\?.*filmId=(?P<id>.+?))(?:&|/|$)'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/films/title/(?P<id>[^/]+)'
+    _TEST = {
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
-        'info_dict':
-        {
+        'md5': '19844f897b35af219773fd63bdec2942',
+        'info_dict': {
             'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
             'display_id': 'lost_for_life',
             'ext': 'mp4',
             'title': 'Lost for Life',
+            'description': 'md5:fbdacc8bb6b455e464aaf98bc02e1c82',
+            'thumbnail': 're:^https?://.*\.jpg',
             'duration': 4489,
-            'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
-            'categories': ['Documentary','Crime','Award Winning','Festivals']
+            'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
         }
-    },{
-        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831',
-        'info_dict':
-        {
-            'id': '74849a00-85a9-11e1-9660-123139220831',
-            'display_id': 'while_we_watch',
-            'ext': 'mp4',
-            'title': '#whilewewatch',
-            'duration': 2311,
-            'description': 'A gripping portrait of the Occupy Wall Street media revolution,&nbsp;#WHILEWEWATCH is the first definitive film to emerge from Zuccotti Park—with full access and cooperation from masterminds who made #OccupyWallStreet a reality.&nbsp;The #OccupyWallStreet media team had no fear of a critical city government, big corporations, hostile police or a lagging mainstream media to tell their story. Through rain, snow, grueling days and sleeping on concrete, they pump out exhilarating ideas to the world. With little money, they rely on Twitter, texting, Wi-Fi, posters, Tumblr, live streams, YouTube, Facebook, dramatic marches, drumbeats and chants. As the film unfolds, we witness the burgeoning power of social media.<br />',
-            'categories': ['Documentary','Politics']
-        }
-    }]
+    }
 
     def _real_extract(self, url):
-        display_id, video_id = match(self._VALID_URL,url).groups()
-        if display_id is None:
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-            display_id = self._html_search_regex(
-                r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
-                embed_webpage,
-                'display_id'
-            )
-        webpage = self._download_webpage('http://www.snagfilms.com/films/title/' + display_id, display_id)
+        display_id = self._match_id(url)
 
-        json_data = self._parse_json(self._html_search_regex(
-            r'"data":{"film":(?P<data>{.*?}})}',
-            webpage,
-            'data'
-        ), display_id)
+        webpage = self._download_webpage(url, display_id)
 
-        if video_id is None:
-            video_id = json_data['id']
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+        film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 
-        title = json_data['title']
-        duration = int(json_data['duration'])
-        description = json_data['synopsis']
-        categories = [category['title'] for category in json_data['categories']]
-        thumbnail = json_data['image']
+        snag = self._parse_json(
+            self._search_regex(
+                'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag'),
+            display_id)
 
-        sources = self._parse_json(js_to_json(self._html_search_regex(
-            r'sources: (?P<sources>\[.*?\])',
-            embed_webpage,
-            'sources',
-            flags=DOTALL
-        )), video_id)
-
-        formats = []
-        for source in sources:
-            if source['type'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
-            else:
-                formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
-        self._sort_formats(formats)
+        for item in snag:
+            if item.get('data', {}).get('film', {}).get('id') == film_id:
+                data = item['data']['film']
+                title = data['title']
+                description = clean_html(data.get('synopsis'))
+                thumbnail = data.get('image')
+                duration = int_or_none(data.get('duration') or data.get('runtime'))
+                categories = [
+                    category['title'] for category in data.get('categories', [])
+                    if category.get('title')]
+                break
+        else:
+            title = self._search_regex(
+                r'itemprop="title">([^<]+)<', webpage, 'title')
+            description = self._html_search_regex(
+                r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
+                webpage, 'description', default=None) or self._og_search_description(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
+            duration = parse_duration(self._search_regex(
+                r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
+                webpage, 'duration', fatal=False))
+            categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'url': 'http://embed.snagfilms.com/embed/player?filmId=%s' % film_id,
+            'id': film_id,
             'display_id': display_id,
             'title': title,
-            'duration': duration,
             'description': description,
-            'categories': categories,
             'thumbnail': thumbnail,
-            'formats': formats,
+            'duration': duration,
+            'categories': categories,
         }

From 7c197ad96dd0f36177eda66777c93502228fc36b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:25:50 +0600
Subject: [PATCH 1015/1515] [snagfilms] Add routine for generic embeds
 extractions

---
 youtube_dl/extractor/snagfilms.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 6e103bd49..cb52eb72b 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -27,6 +27,13 @@ class SnagFilmsEmbedIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From eedd20ef9637660d5585b1b7d221d7f2d31630ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:26:14 +0600
Subject: [PATCH 1016/1515] [extractor/generic] Add support for snagfilms
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 42e4e7035..5b1da47e3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -47,6 +47,7 @@ from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
+from .snagfilms import SnagFilmsEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1550,6 +1551,11 @@ class GenericIE(InfoExtractor):
         if onionstudios_url:
             return self.url_result(onionstudios_url)
 
+        # Look for SnagFilms embeds
+        snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
+        if snagfilms_url:
+            return self.url_result(snagfilms_url)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From b8c1cc1a51c16682725cf382f0e498a390c62a2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:28:10 +0600
Subject: [PATCH 1017/1515] [extractor/generic] Add test for snagfilms embeds

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5b1da47e3..32e41d13e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -850,6 +850,15 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'clickhole',
             }
         },
+        # SnagFilms embed
+        {
+            'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
+            'info_dict': {
+                'id': '74849a00-85a9-11e1-9660-123139220831',
+                'ext': 'mp4',
+                'title': '#whilewewatch',
+            }
+        },
         # AdobeTVVideo embed
         {
             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',

From 242a998bdc5e808411870d7b13395b7739501f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:40:01 +0600
Subject: [PATCH 1018/1515] [snagfilms] Add support for shows

---
 youtube_dl/extractor/snagfilms.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cb52eb72b..f04ca6036 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -76,8 +76,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/films/title/(?P<id>[^/]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/(?:films/title|show)/(?P<id>[^?#]+)'
+    _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'md5': '19844f897b35af219773fd63bdec2942',
         'info_dict': {
@@ -90,7 +90,20 @@ class SnagFilmsIE(InfoExtractor):
             'duration': 4489,
             'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
         }
-    }
+    }, {
+        'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
+        'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
+        'info_dict': {
+            'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
+            'display_id': 'the_world_cut_project/india',
+            'ext': 'mp4',
+            'title': 'India',
+            'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 979,
+            'categories': ['Documentary', 'Sports', 'Politics']
+        }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From 9fbfc9bd4dd076e2b00f68dc43ce4fa0ed72c949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:50:26 +0600
Subject: [PATCH 1019/1515] [snagfilms:embed] Capture geolocation restriction
 error

---
 youtube_dl/extractor/snagfilms.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index f04ca6036..2b1838bf9 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     determine_ext,
     int_or_none,
@@ -30,7 +31,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1',
+            webpage)
         if mobj:
             return mobj.group('url')
 
@@ -39,6 +41,10 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        if '>This film is not playable in your area.<' in webpage:
+            raise ExtractorError(
+                'This film is not playable in your area.', expected=True)
+
         formats = []
         for source in self._parse_json(js_to_json(self._search_regex(
                 r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):

From a42a1bb09dd1c86b4d35eedf1104d8e51b6c407b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:54:08 +0600
Subject: [PATCH 1020/1515] [snagfilms] Capture not available error

---
 youtube_dl/extractor/snagfilms.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 2b1838bf9..1be7dd13b 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -43,7 +43,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
         if '>This film is not playable in your area.<' in webpage:
             raise ExtractorError(
-                'This film is not playable in your area.', expected=True)
+                'Film %s is not playable in your area.' % video_id, expected=True)
 
         formats = []
         for source in self._parse_json(js_to_json(self._search_regex(
@@ -116,6 +116,10 @@ class SnagFilmsIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        if ">Sorry, the Film you're looking for is not available.<" in webpage:
+            raise ExtractorError(
+                'Film %s is not available.' % display_id, expected=True)
+
         film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 
         snag = self._parse_json(

From a9de9517445a4971cb89c73d28a3c7b9ecf51b11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:57:01 +0600
Subject: [PATCH 1021/1515] [snagfilms] More tests

---
 youtube_dl/extractor/snagfilms.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 1be7dd13b..cf495f310 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -109,6 +109,14 @@ class SnagFilmsIE(InfoExtractor):
             'duration': 979,
             'categories': ['Documentary', 'Sports', 'Politics']
         }
+    }, {
+        # Film is not playable in your area.
+        'url': 'http://www.snagfilms.com/films/title/inside_mecca',
+        'only_matching': True,
+    }, {
+        # Film is not available.
+        'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 81103ef35d5148f52229f35e91b99ff8d9218b10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:00:27 +0600
Subject: [PATCH 1022/1515] [smotri] Fix password protected video extraction

---
 youtube_dl/extractor/smotri.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 24746a09a..b8fa0457d 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -152,6 +152,10 @@ class SmotriIE(InfoExtractor):
             'getvideoinfo': '1',
         }
 
+        video_password = self._downloader.params.get('videopassword', None)
+        if video_password:
+            video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
+
         request = compat_urllib_request.Request(
             'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@@ -161,13 +165,18 @@ class SmotriIE(InfoExtractor):
         video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
 
         if not video_url:
-            if video.get('_moderate_no') or not video.get('moderated'):
+            if video.get('_moderate_no'):
                 raise ExtractorError(
                     'Video %s has not been approved by moderator' % video_id, expected=True)
 
             if video.get('error'):
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
+            if video.get('_pass_protected') == 1:
+                msg = ('Invalid video password' if video_password
+                       else 'This video is protected by a password, use the --video-password option')
+                raise ExtractorError(msg, expected=True)
+
         title = video['title']
         thumbnail = video['_imgURL']
         upload_date = unified_strdate(video['added'])

From c3b7202f4fdb8a78a90a96f3cf6213f6ac215b3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:03:26 +0600
Subject: [PATCH 1023/1515] [smotri] Remove non relevant test

---
 youtube_dl/extractor/smotri.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index b8fa0457d..cc8dce3db 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -90,21 +90,6 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
-        # not approved by moderator, but available
-        {
-            'url': 'http://smotri.com/video/view/?id=v28888533b73',
-            'md5': 'f44bc7adac90af518ef1ecf04893bb34',
-            'info_dict': {
-                'id': 'v28888533b73',
-                'ext': 'mp4',
-                'title': 'Russian Spies Killed By ISIL Child Soldier',
-                'uploader': 'Mopeder',
-                'uploader_id': 'mopeder',
-                'duration': 71,
-                'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
-                'upload_date': '20150114',
-            },
-        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',

From 04c27802c08268c5c9e42ca226d2a848a657b82c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:31:27 +0600
Subject: [PATCH 1024/1515] [smotri] Add tests for password protected videos

---
 youtube_dl/extractor/smotri.py | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index cc8dce3db..93a7cfe15 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -53,7 +53,7 @@ class SmotriIE(InfoExtractor):
                 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
             },
         },
-        # video-password
+        # video-password, not approved by moderator
         {
             'url': 'http://smotri.com/video/view/?id=v1390466a13c',
             'md5': 'f6331cef33cad65a0815ee482a54440b',
@@ -71,7 +71,24 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
-        # age limit + video-password
+        # video-password
+        {
+            'url': 'http://smotri.com/video/view/?id=v6984858774#',
+            'md5': 'f11e01d13ac676370fc3b95b9bda11b0',
+            'info_dict': {
+                'id': 'v6984858774',
+                'ext': 'mp4',
+                'title': 'Дача Солженицина ПАРОЛЬ 223322',
+                'uploader': 'psavari1',
+                'uploader_id': 'psavari1',
+                'upload_date': '20081103',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'videopassword': '223322',
+            },
+        },
+        # age limit + video-password, not approved by moderator
         {
             'url': 'http://smotri.com/video/view/?id=v15408898bcf',
             'md5': '91e909c9f0521adf5ee86fbe073aad70',
@@ -90,6 +107,24 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
+        # age limit + video-password
+        {
+            'url': 'http://smotri.com/video/view/?id=v7780025814',
+            'md5': 'b4599b068422559374a59300c5337d72',
+            'info_dict': {
+                'id': 'v7780025814',
+                'ext': 'mp4',
+                'title': 'Sexy Beach (пароль 123)',
+                'uploader': 'вАся',
+                'uploader_id': 'asya_prosto',
+                'upload_date': '20081218',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'age_limit': 18,
+            },
+            'params': {
+                'videopassword': '123'
+            },
+        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',

From 6c4d20cd6fae8e47c1f0c6771e2e91660bf10eb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 00:08:52 +0600
Subject: [PATCH 1025/1515] [downloader/external] Fix externals downloaders
 specified with extension on Windows

---
 youtube_dl/downloader/external.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index a57c15856..1d5cc9904 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -131,5 +131,6 @@ def list_external_downloaders():
 def get_external_downloader(external_downloader):
     """ Given the name of the executable, see whether we support the given
         downloader . """
-    bn = os.path.basename(external_downloader)
+    # Drop .exe extension on Windows
+    bn = os.path.splitext(os.path.basename(external_downloader))[0]
     return _BY_NAME[bn]

From 82ea1051b54134150f290d3d9f259a4baf54faf4 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 05:11:23 +0100
Subject: [PATCH 1026/1515] [moviefap] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/moviefap.py | 117 +++++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 youtube_dl/extractor/moviefap.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7e74a971d..b70c1b57e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -311,6 +311,7 @@ from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
+from .moviefap import MovieFapIE
 from .moviezine import MoviezineIE
 from .movshare import MovShareIE
 from .mtv import (
diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
new file mode 100644
index 000000000..49b8ab7d9
--- /dev/null
+++ b/youtube_dl/extractor/moviefap.py
@@ -0,0 +1,117 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import str_to_int
+
+
+class MovieFapIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
+    _TESTS = [{
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
+            'uploader_id': 'whiskeyjar',
+            'display_id': 'jeune-couple-russe'
+        }
+    }, {
+        'url': 'http://www.moviefap.com/videos/3080837f6712355015c2/busty-british-blonde-takes-backdoor-in-fake-taxi.html',
+        'md5': 'bedef72cb23d27a20755fc430a6d7a0e',
+        'info_dict': {
+            'id': '3080837f6712355015c2',
+            'ext': 'mp4',
+            'title': 'Busty British blonde takes backdoor in fake taxi',
+            'description': 'Big boobs British blonde flashing in fake taxi then giving titsjob and rimjob in the back seat before getting big cock up her tight ass',
+            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/30/322021-18l.jpg',
+            'uploader_id': 'momcikoper',
+            'display_id': 'busty-british-blonde-takes-backdoor-in-fake-taxi'
+        }
+    }]
+
+    @staticmethod
+    def __get_thumbnail_data(xml):
+
+        """
+        Constructs a list of video thumbnails from timeline preview images.
+        :param xml: the information XML document to parse
+        """
+
+        timeline = xml.find('timeline')
+        if timeline is None:
+            # not all videos have the data - ah well
+            return []
+
+        # get the required information from the XML
+        attrs = {attr: str_to_int(timeline.find(attr).text)
+                 for attr in ['imageWidth', 'imageHeight', 'imageFirst', 'imageLast']}
+        pattern = timeline.find('imagePattern').text
+
+        # generate the list of thumbnail information dicts
+        thumbnails = []
+        for i in range(attrs['imageFirst'], attrs['imageLast'] + 1):
+            thumbnails.append({
+                'url': pattern.replace('#', str(i)),
+                'width': attrs['imageWidth'],
+                'height': attrs['imageHeight']
+            })
+        return thumbnails
+
+    def _real_extract(self, url):
+
+        # find the video ID
+        video_id = self._match_id(url)
+
+        # retrieve the page HTML
+        webpage = self._download_webpage(url, video_id)
+
+        # find the URL of the XML document detailing video download URLs
+        info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+
+        # download that XML
+        xml = self._download_xml(info_url, video_id)
+
+        # create dictionary of properties we know so far, or can find easily
+        info = {
+            'id': video_id,
+            'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'display_id': re.compile(self._VALID_URL).match(url).group('name'),
+            'thumbnails': self.__get_thumbnail_data(xml),
+            'thumbnail': xml.find('startThumb').text,
+            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description'),
+            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id'),
+            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count')),
+            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating')),
+            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count')),
+            'age_limit': 18,
+            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url'),
+            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories').split(', ')
+        }
+
+        # find and add the format
+        if xml.find('videoConfig') is not None:
+            info['ext'] = xml.find('videoConfig').find('type').text
+        else:
+            info['ext'] = 'flv'  # guess...
+
+        # work out the video URL(s)
+        if xml.find('videoLink') is not None:
+            # single format available
+            info['url'] = xml.find('videoLink').text
+        else:
+            # multiple formats available
+            info['formats'] = []
+
+            # N.B. formats are already in ascending order of quality
+            for item in xml.find('quality').findall('item'):
+                info['formats'].append({
+                    'url': item.find('videoLink').text,
+                    'resolution': item.find('res').text  # 480p etc.
+                })
+
+        return info

From 71f9e49e67bfe62c9f1a6d8f74b7d81ab820ba84 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 05:22:35 +0100
Subject: [PATCH 1027/1515] [moviefap] Fix dictionary comprehension syntax
 incompatible with Python 2.6

---
 youtube_dl/extractor/moviefap.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 49b8ab7d9..88f9dab6f 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -48,17 +48,19 @@ class MovieFapIE(InfoExtractor):
             return []
 
         # get the required information from the XML
-        attrs = {attr: str_to_int(timeline.find(attr).text)
-                 for attr in ['imageWidth', 'imageHeight', 'imageFirst', 'imageLast']}
+        width = str_to_int(timeline.find('imageWidth').text)
+        height = str_to_int(timeline.find('imageHeight').text)
+        first = str_to_int(timeline.find('imageFirst').text)
+        last = str_to_int(timeline.find('imageLast').text)
         pattern = timeline.find('imagePattern').text
 
         # generate the list of thumbnail information dicts
         thumbnails = []
-        for i in range(attrs['imageFirst'], attrs['imageLast'] + 1):
+        for i in range(first, last + 1):
             thumbnails.append({
                 'url': pattern.replace('#', str(i)),
-                'width': attrs['imageWidth'],
-                'height': attrs['imageHeight']
+                'width': width,
+                'height': height
             })
         return thumbnails
 

From 802d74aa6ba2613f95d19c3be6c2480b9a2ebe5b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:01:27 +0100
Subject: [PATCH 1028/1515] [moviefap] Swap test for an alternative
 non-copyrighted video

---
 youtube_dl/extractor/moviefap.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 88f9dab6f..880ea2247 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -21,16 +21,16 @@ class MovieFapIE(InfoExtractor):
             'display_id': 'jeune-couple-russe'
         }
     }, {
-        'url': 'http://www.moviefap.com/videos/3080837f6712355015c2/busty-british-blonde-takes-backdoor-in-fake-taxi.html',
-        'md5': 'bedef72cb23d27a20755fc430a6d7a0e',
+        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
+        'md5': '26624b4e2523051b550067d547615906',
         'info_dict': {
-            'id': '3080837f6712355015c2',
+            'id': 'be9867c9416c19f54a4a',
             'ext': 'mp4',
-            'title': 'Busty British blonde takes backdoor in fake taxi',
-            'description': 'Big boobs British blonde flashing in fake taxi then giving titsjob and rimjob in the back seat before getting big cock up her tight ass',
-            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/30/322021-18l.jpg',
-            'uploader_id': 'momcikoper',
-            'display_id': 'busty-british-blonde-takes-backdoor-in-fake-taxi'
+            'title': 'Experienced MILF Amazing Handjob',
+            'description': 'Experienced MILF giving an Amazing Handjob',
+            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
+            'uploader_id': 'darvinfred06',
+            'display_id': 'experienced-milf-amazing-handjob'
         }
     }]
 

From 9c494108983c2a951780217014c26d8f3b081682 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:06:44 +0100
Subject: [PATCH 1029/1515] [moviefap] Add categories to tests

---
 youtube_dl/extractor/moviefap.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 880ea2247..6b79183c7 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -18,7 +18,8 @@ class MovieFapIE(InfoExtractor):
             'description': 'Amateur',
             'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
             'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe'
+            'display_id': 'jeune-couple-russe',
+            'categories': ['Amateur', 'Teen']
         }
     }, {
         'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
@@ -30,7 +31,8 @@ class MovieFapIE(InfoExtractor):
             'description': 'Experienced MILF giving an Amazing Handjob',
             'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
             'uploader_id': 'darvinfred06',
-            'display_id': 'experienced-milf-amazing-handjob'
+            'display_id': 'experienced-milf-amazing-handjob',
+            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
         }
     }]
 

From a8e6f30d8ed13209f2e9815ae97b3edccd38561b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:14:15 +0100
Subject: [PATCH 1030/1515] [moviefap] Swap and justify tests

---
 youtube_dl/extractor/moviefap.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 6b79183c7..1985c53c0 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -9,19 +9,7 @@ from ..utils import str_to_int
 class MovieFapIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
     _TESTS = [{
-        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
-        'md5': 'fa56683e291fc80635907168a743c9ad',
-        'info_dict': {
-            'id': 'e5da0d3edce5404418f5',
-            'ext': 'flv',
-            'title': 'Jeune Couple Russe',
-            'description': 'Amateur',
-            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
-            'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe',
-            'categories': ['Amateur', 'Teen']
-        }
-    }, {
+        # normal, multi-format video
         'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
         'md5': '26624b4e2523051b550067d547615906',
         'info_dict': {
@@ -34,6 +22,20 @@ class MovieFapIE(InfoExtractor):
             'display_id': 'experienced-milf-amazing-handjob',
             'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
         }
+    }, {
+        # quirky single-format case where the extension is given as fid, but the video is really an flv
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
+            'uploader_id': 'whiskeyjar',
+            'display_id': 'jeune-couple-russe',
+            'categories': ['Amateur', 'Teen']
+        }
     }]
 
     @staticmethod

From 23e7cba87fbcec9aa2b1fbccf60c2d560df3d7ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 01:22:25 +0600
Subject: [PATCH 1031/1515] [twitter:card] Add extractor (#5239)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitter.py  | 72 ++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/twitter.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7e74a971d..c0a825e8a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -621,6 +621,7 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
+from .twitter import TwitterCardIE
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
new file mode 100644
index 000000000..1aaa06305
--- /dev/null
+++ b/youtube_dl/extractor/twitter.py
@@ -0,0 +1,72 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    float_or_none,
+    unescapeHTML,
+)
+
+
+class TwitterCardIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
+        'md5': 'a74f50b310c83170319ba16de6955192',
+        'info_dict': {
+            'id': '560070183650213889',
+            'ext': 'mp4',
+            'title': 'TwitterCard',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 30.033,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Different formats served for different User-Agents
+        USER_AGENTS = [
+            'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',  # mp4
+            'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',  # webm
+        ]
+
+        config = None
+        formats = []
+        for user_agent in USER_AGENTS:
+            request = compat_urllib_request.Request(url)
+            request.add_header('User-Agent', user_agent)
+            webpage = self._download_webpage(request, video_id)
+
+            config = self._parse_json(
+                unescapeHTML(self._search_regex(
+                    r'data-player-config="([^"]+)"', webpage, 'data player config')),
+                video_id)
+
+            video_url = config['playlist'][0]['source']
+
+            f = {
+                'url': video_url,
+            }
+
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                f.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+            formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnail = config.get('posterImageUrl')
+        duration = float_or_none(config.get('duration'))
+
+        return {
+            'id': video_id,
+            'title': 'TwitterCard',
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From d16ef949ca94ffb998c4db5a8367f0b367d659e3 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:36:46 +0100
Subject: [PATCH 1032/1515] [moviefap] Allow non-critical fields to change
 without breaking extraction

---
 youtube_dl/extractor/moviefap.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 1985c53c0..23575d30a 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -89,14 +89,14 @@ class MovieFapIE(InfoExtractor):
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xml.find('startThumb').text,
-            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description'),
-            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id'),
-            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count')),
-            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating')),
-            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count')),
+            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url'),
-            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories').split(', ')
+            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
 
         # find and add the format

From 62b742ece3ec6c7d7fd24898b5413b6b98a4ae8f Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:51:11 +0100
Subject: [PATCH 1033/1515] [moviefap] Remove redundant comments

---
 youtube_dl/extractor/moviefap.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 23575d30a..b38a8e71f 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -70,19 +70,13 @@ class MovieFapIE(InfoExtractor):
 
     def _real_extract(self, url):
 
-        # find the video ID
         video_id = self._match_id(url)
-
-        # retrieve the page HTML
         webpage = self._download_webpage(url, video_id)
 
-        # find the URL of the XML document detailing video download URLs
+        # find and retrieve the XML document detailing video download URLs
         info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
-
-        # download that XML
         xml = self._download_xml(info_url, video_id)
 
-        # create dictionary of properties we know so far, or can find easily
         info = {
             'id': video_id,
             'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),

From 43b925ce74efd0a011f7880dcdcc90f4cf3b8f4b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:52:12 +0100
Subject: [PATCH 1034/1515] [moviefap] Replace calls to `find()` with
 `util.xpath_text()`.

---
 youtube_dl/extractor/moviefap.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index b38a8e71f..6da93dbc9 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import str_to_int
+from ..utils import (
+    xpath_text,
+    str_to_int
+)
 
 
 class MovieFapIE(InfoExtractor):
@@ -82,7 +85,7 @@ class MovieFapIE(InfoExtractor):
             'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
-            'thumbnail': xml.find('startThumb').text,
+            'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
             'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
             'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
             'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
@@ -102,7 +105,7 @@ class MovieFapIE(InfoExtractor):
         # work out the video URL(s)
         if xml.find('videoLink') is not None:
             # single format available
-            info['url'] = xml.find('videoLink').text
+            info['url'] = xpath_text(xml, 'videoLink', 'url', True)
         else:
             # multiple formats available
             info['formats'] = []
@@ -110,8 +113,8 @@ class MovieFapIE(InfoExtractor):
             # N.B. formats are already in ascending order of quality
             for item in xml.find('quality').findall('item'):
                 info['formats'].append({
-                    'url': item.find('videoLink').text,
-                    'resolution': item.find('res').text  # 480p etc.
+                    'url': xpath_text(item, 'videoLink', 'url', True),
+                    'resolution': xpath_text(item, 'res', 'resolution', True)  # 480p etc.
                 })
 
         return info

From b971abe897ee17fed7e36868fdc8880f6b145d7b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 21:04:53 +0100
Subject: [PATCH 1035/1515] [moviefap] Replace call to `str()` with
 `compat.compat_str()`

---
 youtube_dl/extractor/moviefap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 6da93dbc9..20a78f3b2 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -7,6 +7,7 @@ from ..utils import (
     xpath_text,
     str_to_int
 )
+from ..compat import compat_str
 
 
 class MovieFapIE(InfoExtractor):
@@ -65,7 +66,7 @@ class MovieFapIE(InfoExtractor):
         thumbnails = []
         for i in range(first, last + 1):
             thumbnails.append({
-                'url': pattern.replace('#', str(i)),
+                'url': pattern.replace('#', compat_str(i)),
                 'width': width,
                 'height': height
             })

From 8a1b49ff19a8a1fdc2c30cf10cc0598ac9bc8819 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 22:27:06 +0100
Subject: [PATCH 1036/1515] [moviefap] Explicitly sort formats to handle
 possible site changes

---
 youtube_dl/extractor/moviefap.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 20a78f3b2..295bfe3f0 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -111,11 +111,14 @@ class MovieFapIE(InfoExtractor):
             # multiple formats available
             info['formats'] = []
 
-            # N.B. formats are already in ascending order of quality
             for item in xml.find('quality').findall('item'):
+                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
                 info['formats'].append({
                     'url': xpath_text(item, 'videoLink', 'url', True),
-                    'resolution': xpath_text(item, 'res', 'resolution', True)  # 480p etc.
+                    'resolution': resolution,
+                    'height': int(re.findall(r'\d+', resolution)[0])
                 })
 
+            self._sort_formats(info['formats'])
+
         return info

From 1a5fd4eebc2717b5173df50d65007f90cb05ee30 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 22:32:56 +0100
Subject: [PATCH 1037/1515] [moviefap] Wrap long lines

---
 youtube_dl/extractor/moviefap.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 295bfe3f0..9de052a99 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -78,23 +78,32 @@ class MovieFapIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+        info_url = self._html_search_regex( \
+                r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
         info = {
             'id': video_id,
-            'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'title': self._html_search_regex( \
+                    r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
+            'description': self._html_search_regex( \
+                    r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex( \
+                    r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex( \
+                    r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex( \
+                    r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex( \
+                    r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
+            'webpage_url': self._html_search_regex( \
+                    r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex( \
+                    r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
 
         # find and add the format

From 5a9cc19972fb3aae7a67470f65ec5cd30918f4e1 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 23:03:06 +0100
Subject: [PATCH 1038/1515] [moviefap] Move flv videos to formats in the
 metadata

---
 youtube_dl/extractor/moviefap.py | 56 +++++++++++++++++---------------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 9de052a99..5e0c701d4 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -82,8 +82,36 @@ class MovieFapIE(InfoExtractor):
                 r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
-        info = {
+        # find the video container
+        if xml.find('videoConfig') is not None:
+            ext = xml.find('videoConfig').find('type').text
+        else:
+            ext = 'flv'  # guess...
+
+        # work out the video URL(s)
+        formats = []
+        if xml.find('videoLink') is not None:
+            # single format available
+            formats.append({
+                'url': xpath_text(xml, 'videoLink', 'url', True),
+                'ext': ext
+            })
+        else:
+            # multiple formats available
+            for item in xml.find('quality').findall('item'):
+                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
+                formats.append({
+                    'url': xpath_text(item, 'videoLink', 'url', True),
+                    'ext': ext,
+                    'resolution': resolution,
+                    'height': int(re.findall(r'\d+', resolution)[0])
+                })
+
+            self._sort_formats(formats)
+
+        return {
             'id': video_id,
+            'formats': formats,
             'title': self._html_search_regex( \
                     r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
@@ -105,29 +133,3 @@ class MovieFapIE(InfoExtractor):
             'categories': self._html_search_regex( \
                     r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
-
-        # find and add the format
-        if xml.find('videoConfig') is not None:
-            info['ext'] = xml.find('videoConfig').find('type').text
-        else:
-            info['ext'] = 'flv'  # guess...
-
-        # work out the video URL(s)
-        if xml.find('videoLink') is not None:
-            # single format available
-            info['url'] = xpath_text(xml, 'videoLink', 'url', True)
-        else:
-            # multiple formats available
-            info['formats'] = []
-
-            for item in xml.find('quality').findall('item'):
-                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
-                info['formats'].append({
-                    'url': xpath_text(item, 'videoLink', 'url', True),
-                    'resolution': resolution,
-                    'height': int(re.findall(r'\d+', resolution)[0])
-                })
-
-            self._sort_formats(info['formats'])
-
-        return info

From db652ea186586e3eda5006ee096161b1a867c0d0 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 23:04:55 +0100
Subject: [PATCH 1039/1515] [moviefap] Fix `flake8` warnings introduced in
 1a5fd4e

---
 youtube_dl/extractor/moviefap.py | 36 ++++++++++++++++----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 5e0c701d4..82b863539 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -78,8 +78,8 @@ class MovieFapIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex( \
-                r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+        info_url = self._html_search_regex(
+            r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
         # find the video container
@@ -112,24 +112,24 @@ class MovieFapIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': self._html_search_regex( \
-                    r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'title': self._html_search_regex(
+                r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex( \
-                    r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex( \
-                    r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex( \
-                    r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex( \
-                    r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex( \
-                    r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
+            'description': self._html_search_regex(
+                r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex(
+                r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex(
+                r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex(
+                r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex(
+                r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex( \
-                    r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex( \
-                    r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
+            'webpage_url': self._html_search_regex(
+                r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex(
+                r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }

From bb512e57dc138b261cf9c71a833b0df5d5ba849f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Jun 2015 13:25:59 +0800
Subject: [PATCH 1040/1515] [twitch:vod] Fix 'Source' format in m3u8 (closes
 #6115)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 94bd6345d..3e798e62d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -215,7 +215,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         formats = self._extract_m3u8_formats(
-            '%s/vod/%s?nauth=%s&nauthsig=%s'
+            '%s/vod/%s?nauth=%s&nauthsig=%s&allow_source=true'
             % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
         self._prefer_source(formats)

From ac0474f89d3e6f8c8c1fb3223a16a18a2fd02bcb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Jun 2015 13:31:37 +0800
Subject: [PATCH 1041/1515] [twitch:vod] Update _TEST

The original test case is gone
---
 youtube_dl/extractor/twitch.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 3e798e62d..b56ee2959 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -189,17 +189,17 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_SHORTCUT = 'v'
 
     _TEST = {
-        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+        'url': 'http://www.twitch.tv/riotgames/v/6528877',
         'info_dict': {
-            'id': 'v3622000',
+            'id': 'v6528877',
             'ext': 'mp4',
-            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'title': 'LCK Summer Split - Week 6 Day 1',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 6951,
-            'timestamp': 1419028564,
-            'upload_date': '20141219',
-            'uploader': 'KSPTV',
-            'uploader_id': 'ksptv',
+            'duration': 17208,
+            'timestamp': 1435131709,
+            'upload_date': '20150624',
+            'uploader': 'Riot Games',
+            'uploader_id': 'riotgames',
             'view_count': int,
         },
         'params': {

From 9603e8a7d998615d3da1af47461ec9c353ec4e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:55:28 +0600
Subject: [PATCH 1042/1515] [YoutubeDL] Handle None width and height similarly
 to formats

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ef0f71bad..411de9ac9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1008,7 +1008,7 @@ class YoutubeDL(object):
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
             for i, t in enumerate(thumbnails):
-                if 'width' in t and 'height' in t:
+                if t.get('width') and t.get('height'):
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
                 if t.get('id') is None:
                     t['id'] = '%d' % i

From bf42a9906d9a066d32f1cc50e1b033e6676744ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:56:07 +0600
Subject: [PATCH 1043/1515] [utils] Add default value for xpath_text

---
 youtube_dl/utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 96490f112..942f76d24 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -62,6 +62,8 @@ std_headers = {
 }
 
 
+NO_DEFAULT = object()
+
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
@@ -171,13 +173,15 @@ def xpath_with_ns(path, ns_map):
     return '/'.join(replaced)
 
 
-def xpath_text(node, xpath, name=None, fatal=False):
+def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     if sys.version_info < (2, 7):  # Crazy 2.6
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
     if n is None or n.text is None:
-        if fatal:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
             name = xpath if name is None else name
             raise ExtractorError('Could not find XML element %s' % name)
         else:

From c342041fba9283ba5f05f48427aabf79adcf8647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:56:45 +0600
Subject: [PATCH 1044/1515] [extractor/common] Use NO_DEFAULT from utils

---
 youtube_dl/extractor/common.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 49e4dc710..7fa46d295 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -22,6 +22,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
+    NO_DEFAULT,
     age_restricted,
     bug_reports_message,
     clean_html,
@@ -33,7 +34,7 @@ from ..utils import (
     sanitize_filename,
     unescapeHTML,
 )
-_NO_DEFAULT = object()
+
 
 
 class InfoExtractor(object):
@@ -523,7 +524,7 @@ class InfoExtractor(object):
             video_info['description'] = playlist_description
         return video_info
 
-    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
+    def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Perform a regex search on the given string, using a single or a list of
         patterns returning the first matching group.
@@ -549,7 +550,7 @@ class InfoExtractor(object):
                 return next(g for g in mobj.groups() if g is not None)
             else:
                 return mobj.group(group)
-        elif default is not _NO_DEFAULT:
+        elif default is not NO_DEFAULT:
             return default
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
@@ -557,7 +558,7 @@ class InfoExtractor(object):
             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
-    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
+    def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Like _search_regex, but strips HTML tags and unescapes entities.
         """

From d16154d16327907279eff48a4018c495726d401a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 23:05:09 +0600
Subject: [PATCH 1045/1515] [tnaflix] Generalize tnaflix extractors

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/empflix.py  |  31 ----
 youtube_dl/extractor/moviefap.py | 135 ---------------
 youtube_dl/extractor/tnaflix.py  | 279 +++++++++++++++++++++++++------
 4 files changed, 234 insertions(+), 219 deletions(-)
 delete mode 100644 youtube_dl/extractor/empflix.py
 delete mode 100644 youtube_dl/extractor/moviefap.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d41d277c9..d44339200 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -144,7 +144,6 @@ from .ellentv import (
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE
-from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
@@ -311,7 +310,6 @@ from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
-from .moviefap import MovieFapIE
 from .moviezine import MoviezineIE
 from .movshare import MovShareIE
 from .mtv import (
@@ -578,7 +576,11 @@ from .tmz import (
     TMZIE,
     TMZArticleIE,
 )
-from .tnaflix import TNAFlixIE
+from .tnaflix import (
+    TNAFlixIE,
+    EMPFlixIE,
+    MovieFapIE,
+)
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
deleted file mode 100644
index 4827022e0..000000000
--- a/youtube_dl/extractor/empflix.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from __future__ import unicode_literals
-
-from .tnaflix import TNAFlixIE
-
-
-class EMPFlixIE(TNAFlixIE):
-    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
-
-    _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
-    _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
-    _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-
-    _TESTS = [
-        {
-            'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-            'md5': 'b1bc15b6412d33902d6e5952035fcabc',
-            'info_dict': {
-                'id': '33051',
-                'display_id': 'Amateur-Finger-Fuck',
-                'ext': 'mp4',
-                'title': 'Amateur Finger Fuck',
-                'description': 'Amateur solo finger fucking.',
-                'thumbnail': 're:https?://.*\.jpg$',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
-            'only_matching': True,
-        }
-    ]
diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
deleted file mode 100644
index 82b863539..000000000
--- a/youtube_dl/extractor/moviefap.py
+++ /dev/null
@@ -1,135 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    str_to_int
-)
-from ..compat import compat_str
-
-
-class MovieFapIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
-    _TESTS = [{
-        # normal, multi-format video
-        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
-        'md5': '26624b4e2523051b550067d547615906',
-        'info_dict': {
-            'id': 'be9867c9416c19f54a4a',
-            'ext': 'mp4',
-            'title': 'Experienced MILF Amazing Handjob',
-            'description': 'Experienced MILF giving an Amazing Handjob',
-            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
-            'uploader_id': 'darvinfred06',
-            'display_id': 'experienced-milf-amazing-handjob',
-            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
-        }
-    }, {
-        # quirky single-format case where the extension is given as fid, but the video is really an flv
-        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
-        'md5': 'fa56683e291fc80635907168a743c9ad',
-        'info_dict': {
-            'id': 'e5da0d3edce5404418f5',
-            'ext': 'flv',
-            'title': 'Jeune Couple Russe',
-            'description': 'Amateur',
-            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
-            'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe',
-            'categories': ['Amateur', 'Teen']
-        }
-    }]
-
-    @staticmethod
-    def __get_thumbnail_data(xml):
-
-        """
-        Constructs a list of video thumbnails from timeline preview images.
-        :param xml: the information XML document to parse
-        """
-
-        timeline = xml.find('timeline')
-        if timeline is None:
-            # not all videos have the data - ah well
-            return []
-
-        # get the required information from the XML
-        width = str_to_int(timeline.find('imageWidth').text)
-        height = str_to_int(timeline.find('imageHeight').text)
-        first = str_to_int(timeline.find('imageFirst').text)
-        last = str_to_int(timeline.find('imageLast').text)
-        pattern = timeline.find('imagePattern').text
-
-        # generate the list of thumbnail information dicts
-        thumbnails = []
-        for i in range(first, last + 1):
-            thumbnails.append({
-                'url': pattern.replace('#', compat_str(i)),
-                'width': width,
-                'height': height
-            })
-        return thumbnails
-
-    def _real_extract(self, url):
-
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex(
-            r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
-        xml = self._download_xml(info_url, video_id)
-
-        # find the video container
-        if xml.find('videoConfig') is not None:
-            ext = xml.find('videoConfig').find('type').text
-        else:
-            ext = 'flv'  # guess...
-
-        # work out the video URL(s)
-        formats = []
-        if xml.find('videoLink') is not None:
-            # single format available
-            formats.append({
-                'url': xpath_text(xml, 'videoLink', 'url', True),
-                'ext': ext
-            })
-        else:
-            # multiple formats available
-            for item in xml.find('quality').findall('item'):
-                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
-                formats.append({
-                    'url': xpath_text(item, 'videoLink', 'url', True),
-                    'ext': ext,
-                    'resolution': resolution,
-                    'height': int(re.findall(r'\d+', resolution)[0])
-                })
-
-            self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': self._html_search_regex(
-                r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
-            'display_id': re.compile(self._VALID_URL).match(url).group('name'),
-            'thumbnails': self.__get_thumbnail_data(xml),
-            'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex(
-                r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex(
-                r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex(
-                r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex(
-                r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex(
-                r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
-            'age_limit': 18,
-            'webpage_url': self._html_search_regex(
-                r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex(
-                r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
-        }
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index c282865b2..49516abca 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -3,39 +3,70 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    parse_duration,
     fix_xml_ampersands,
+    float_or_none,
+    int_or_none,
+    parse_duration,
+    str_to_int,
+    xpath_text,
 )
 
 
-class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
-
-    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
-    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
-    _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-
-    _TESTS = [
-        {
-            'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-            'md5': 'ecf3498417d09216374fc5907f9c6ec0',
-            'info_dict': {
-                'id': '553878',
-                'display_id': 'Carmella-Decesare-striptease',
-                'ext': 'mp4',
-                'title': 'Carmella Decesare - striptease',
-                'description': '',
-                'thumbnail': 're:https?://.*\.jpg$',
-                'duration': 91,
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-            'only_matching': True,
-        }
+class TNAFlixNetworkBaseIE(InfoExtractor):
+    # May be overridden in descendants if necessary
+    _CONFIG_REGEX = [
+        r'flashvars\.config\s*=\s*escape\("([^"]+)"',
+        r'<input[^>]+name="config\d?" value="([^"]+)"',
     ]
+    _TITLE_REGEX = r'<input[^>]+name="title" value="([^"]+)"'
+    _DESCRIPTION_REGEX = r'<input[^>]+name="description" value="([^"]+)"'
+    _UPLOADER_REGEX = r'<input[^>]+name="username" value="([^"]+)"'
+    _VIEW_COUNT_REGEX = None
+    _COMMENT_COUNT_REGEX = None
+    _AVERAGE_RATING_REGEX = None
+    _CATEGORIES_REGEX = r'<li[^>]*>\s*<span[^>]+class="infoTitle"[^>]*>Categories:</span>\s*<span[^>]+class="listView"[^>]*>(.+?)</span>\s*</li>'
+
+    def _extract_thumbnails(self, flix_xml):
+
+        def get_child(elem, names):
+            for name in names:
+                child = elem.find(name)
+                if child is not None:
+                    return child
+
+        timeline = get_child(flix_xml, ['timeline', 'rolloverBarImage'])
+        if timeline is None:
+            return
+
+        pattern_el = get_child(timeline, ['imagePattern', 'pattern'])
+        if pattern_el is None or not pattern_el.text:
+            return
+
+        first_el = get_child(timeline, ['imageFirst', 'first'])
+        last_el = get_child(timeline, ['imageLast', 'last'])
+        if first_el is None or last_el is None:
+            return
+
+        first_text = first_el.text
+        last_text = last_el.text
+        if not first_text.isdigit() or not last_text.isdigit():
+            return
+
+        first = int(first_text)
+        last = int(last_text)
+        if first > last:
+            return
+
+        width = int_or_none(xpath_text(timeline, './imageWidth', 'thumbnail width'))
+        height = int_or_none(xpath_text(timeline, './imageHeight', 'thumbnail height'))
+
+        return [{
+            'url': self._proto_relative_url(pattern_el.text.replace('#', compat_str(i)), 'http:'),
+            'width': width,
+            'height': height,
+        } for i in range(first, last + 1)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -44,39 +75,64 @@ class TNAFlixIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        cfg_url = self._proto_relative_url(self._html_search_regex(
+            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+
+        cfg_xml = self._download_xml(
+            cfg_url, display_id, 'Downloading metadata',
+            transform_source=fix_xml_ampersands)
+
+        formats = []
+
+        def extract_video_url(vl):
+            return re.sub('speed=\d+', 'speed=', vl.text)
+
+        video_link = cfg_xml.find('./videoLink')
+        if video_link is not None:
+            formats.append({
+                'url': extract_video_url(video_link),
+                'ext': xpath_text(cfg_xml, './videoConfig/type', 'type', default='flv'),
+            })
+
+        for item in cfg_xml.findall('./quality/item'):
+            video_link = item.find('./videoLink')
+            if video_link is None:
+                continue
+            res = item.find('res')
+            format_id = None if res is None else res.text
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]', format_id, 'height', default=None))
+            formats.append({
+                'url': self._proto_relative_url(extract_video_url(video_link), 'http:'),
+                'format_id': format_id,
+                'height': height,
+            })
+
+        self._sort_formats(formats)
+
+        thumbnail = self._proto_relative_url(
+            xpath_text(cfg_xml, './startThumb', 'thumbnail'), 'http:')
+        thumbnails = self._extract_thumbnails(cfg_xml)
+
         title = self._html_search_regex(
             self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
-        description = self._html_search_regex(
-            self._DESCRIPTION_REGEX, webpage, 'description', fatal=False, default='')
 
         age_limit = self._rta_search(webpage)
 
         duration = parse_duration(self._html_search_meta(
             'duration', webpage, 'duration', default=None))
 
-        cfg_url = self._proto_relative_url(self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+        def extract_field(pattern, name):
+            return self._html_search_regex(pattern, webpage, name, default=None) if pattern else None
 
-        cfg_xml = self._download_xml(
-            cfg_url, display_id, note='Downloading metadata',
-            transform_source=fix_xml_ampersands)
+        description = extract_field(self._DESCRIPTION_REGEX, 'description')
+        uploader = extract_field(self._UPLOADER_REGEX, 'uploader')
+        view_count = str_to_int(extract_field(self._VIEW_COUNT_REGEX, 'view count'))
+        comment_count = str_to_int(extract_field(self._COMMENT_COUNT_REGEX, 'comment count'))
+        average_rating = float_or_none(extract_field(self._AVERAGE_RATING_REGEX, 'average rating'))
 
-        thumbnail = self._proto_relative_url(
-            cfg_xml.find('./startThumb').text, 'http:')
-
-        formats = []
-        for item in cfg_xml.findall('./quality/item'):
-            video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
-            format_id = item.find('res').text
-            fmt = {
-                'url': self._proto_relative_url(video_url, 'http:'),
-                'format_id': format_id,
-            }
-            m = re.search(r'^(\d+)', format_id)
-            if m:
-                fmt['height'] = int(m.group(1))
-            formats.append(fmt)
-        self._sort_formats(formats)
+        categories_str = extract_field(self._CATEGORIES_REGEX, 'categories')
+        categories = categories_str.split(', ') if categories_str is not None else []
 
         return {
             'id': video_id,
@@ -84,7 +140,130 @@ class TNAFlixIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'duration': duration,
             'age_limit': age_limit,
+            'uploader': uploader,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'average_rating': average_rating,
+            'categories': categories,
             'formats': formats,
         }
+
+
+class TNAFlixIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
+
+    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
+    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
+    _UPLOADER_REGEX = r'(?s)<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)<div'
+
+    _TESTS = [{
+        # anonymous uploader, no categories
+        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'info_dict': {
+            'id': '553878',
+            'display_id': 'Carmella-Decesare-striptease',
+            'ext': 'mp4',
+            'title': 'Carmella Decesare - striptease',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 91,
+            'age_limit': 18,
+            'uploader': 'Anonymous',
+            'categories': [],
+        }
+    }, {
+        # non-anonymous uploader, categories
+        'url': 'https://www.tnaflix.com/teen-porn/Educational-xxx-video/video6538',
+        'md5': '0f5d4d490dbfd117b8607054248a07c0',
+        'info_dict': {
+            'id': '6538',
+            'display_id': 'Educational-xxx-video',
+            'ext': 'mp4',
+            'title': 'Educational xxx video',
+            'description': 'md5:b4fab8f88a8621c8fabd361a173fe5b8',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 164,
+            'age_limit': 18,
+            'uploader': 'bobwhite39',
+            'categories': ['Amateur Porn', 'Squirting Videos', 'Teen Girls 18+'],
+        }
+    }, {
+        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+        'only_matching': True,
+    }]
+
+
+class EMPFlixIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
+
+    _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
+
+    _TESTS = [{
+        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
+        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+        'info_dict': {
+            'id': '33051',
+            'display_id': 'Amateur-Finger-Fuck',
+            'ext': 'mp4',
+            'title': 'Amateur Finger Fuck',
+            'description': 'Amateur solo finger fucking.',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 83,
+            'age_limit': 18,
+            'uploader': 'cwbike',
+            'categories': ['Amateur', 'Anal', 'Fisting', 'Home made', 'Solo'],
+        }
+    }, {
+        'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
+        'only_matching': True,
+    }]
+
+
+class MovieFapIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<display_id>[^/]+)\.html'
+
+    _VIEW_COUNT_REGEX = r'<br>Views\s*<strong>([\d,.]+)</strong>'
+    _COMMENT_COUNT_REGEX = r'<span[^>]+id="comCount"[^>]*>([\d,.]+)</span>'
+    _AVERAGE_RATING_REGEX = r'Current Rating\s*<br>\s*<strong>([\d.]+)</strong>'
+    _CATEGORIES_REGEX = r'(?s)<div[^>]+id="vid_info"[^>]*>\s*<div[^>]*>.+?</div>(.*?)<br>'
+
+    _TESTS = [{
+        # normal, multi-format video
+        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
+        'md5': '26624b4e2523051b550067d547615906',
+        'info_dict': {
+            'id': 'be9867c9416c19f54a4a',
+            'display_id': 'experienced-milf-amazing-handjob',
+            'ext': 'mp4',
+            'title': 'Experienced MILF Amazing Handjob',
+            'description': 'Experienced MILF giving an Amazing Handjob',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+            'uploader': 'darvinfred06',
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing'],
+        }
+    }, {
+        # quirky single-format case where the extension is given as fid, but the video is really an flv
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'display_id': 'jeune-couple-russe',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+            'uploader': 'whiskeyjar',
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Amateur', 'Teen'],
+        }
+    }]

From 507683780eb14d012d4430044dc402d7e08e36b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 23:08:05 +0600
Subject: [PATCH 1046/1515] Credit @gebn for moviefap

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 889d599a2..117b9c219 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -128,3 +128,4 @@ Ping O.
 Mister Hat
 Peter Ding
 jackyzy823
+George Brighton

From c93d53f5e307dab1b5d03cd3c621a68f40ef840d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 00:48:06 +0600
Subject: [PATCH 1047/1515] [youtube] Fix likes/dislike extraction

---
 youtube_dl/extractor/youtube.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d9240ff02..8b43e274b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -29,6 +29,7 @@ from ..utils import (
     get_element_by_id,
     int_or_none,
     orderedSet,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
     uppercase_escape,
@@ -1005,12 +1006,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 video_description = ''
 
         def _extract_count(count_name):
-            count = self._search_regex(
-                r'id="watch-%s"[^>]*>.*?([\d,]+)\s*</span>' % re.escape(count_name),
-                video_webpage, count_name, default=None)
-            if count is not None:
-                return int(count.replace(',', ''))
-            return None
+            return str_to_int(self._search_regex(
+                r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
+                % re.escape(count_name),
+                video_webpage, count_name, default=None))
+
         like_count = _extract_count('like')
         dislike_count = _extract_count('dislike')
 

From 541462379153c19656aa52cc5796dbf05de874ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 00:49:19 +0600
Subject: [PATCH 1048/1515] [extractor/common] Remove superfluous line

---
 youtube_dl/extractor/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7fa46d295..81623bfe3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -36,7 +36,6 @@ from ..utils import (
 )
 
 
-
 class InfoExtractor(object):
     """Information Extractor class.
 

From 67134eaba1a56cec4117000acb2fc9284c9cdd9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Jun 2015 22:08:29 +0200
Subject: [PATCH 1049/1515] [YoutubeDL] rework how the format spec is processed

The spec string is processed using 'tokenize.tokenize' to split it in words and operators, the filters are still processed using regular expressions.
This should make easier to allow grouping operators with parens.
---
 test/test_YoutubeDL.py  |  27 ++--
 youtube_dl/YoutubeDL.py | 298 +++++++++++++++++++++++++---------------
 youtube_dl/compat.py    |   5 +
 3 files changed, 209 insertions(+), 121 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index a13c09ef4..8f7aef512 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -229,21 +229,30 @@ class TestFormatSelection(unittest.TestCase):
             '141', '172', '140', '171', '139',
         ]
 
-        for f1id, f2id in zip(order, order[1:]):
-            f1 = YoutubeIE._formats[f1id].copy()
-            f1['format_id'] = f1id
-            f1['url'] = 'url:' + f1id
-            f2 = YoutubeIE._formats[f2id].copy()
-            f2['format_id'] = f2id
-            f2['url'] = 'url:' + f2id
+        def format_info(f_id):
+            info = YoutubeIE._formats[f_id].copy()
+            info['format_id'] = f_id
+            info['url'] = 'url:' + f_id
+            return info
+        formats_order = [format_info(f_id) for f_id in order]
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '137+141')
+        self.assertEqual(downloaded['ext'], 'mp4')
+
+        for f1, f2 in zip(formats_order, formats_order[1:]):
             info_dict = _make_result([f1, f2], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
             downloaded = ydl.downloaded_info_dicts[0]
-            self.assertEqual(downloaded['format_id'], f1id)
+            self.assertEqual(downloaded['format_id'], f1['format_id'])
 
             info_dict = _make_result([f2, f1], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
@@ -251,7 +260,7 @@ class TestFormatSelection(unittest.TestCase):
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
             downloaded = ydl.downloaded_info_dicts[0]
-            self.assertEqual(downloaded['format_id'], f1id)
+            self.assertEqual(downloaded['format_id'], f1['format_id'])
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ef0f71bad..17a5407b9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -21,6 +21,7 @@ import subprocess
 import socket
 import sys
 import time
+import tokenize
 import traceback
 
 if os.name == 'nt':
@@ -34,6 +35,7 @@ from .compat import (
     compat_http_client,
     compat_kwargs,
     compat_str,
+    compat_tokenize_tokenize,
     compat_urllib_error,
     compat_urllib_request,
 )
@@ -851,8 +853,8 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
-    def _apply_format_filter(self, format_spec, available_formats):
-        " Returns a tuple of the remaining format_spec and filtered formats "
+    def _build_format_filter(self, filter_spec):
+        " Returns a function to filter the formats according to the filter_spec "
 
         OPERATORS = {
             '<': operator.lt,
@@ -862,13 +864,13 @@ class YoutubeDL(object):
             '=': operator.eq,
             '!=': operator.ne,
         }
-        operator_rex = re.compile(r'''(?x)\s*\[
+        operator_rex = re.compile(r'''(?x)\s*
             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
-            \]$
+            $
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
-        m = operator_rex.search(format_spec)
+        m = operator_rex.search(filter_spec)
         if m:
             try:
                 comparison_value = int(m.group('value'))
@@ -879,7 +881,7 @@ class YoutubeDL(object):
                 if comparison_value is None:
                     raise ValueError(
                         'Invalid value %r in format specification %r' % (
-                            m.group('value'), format_spec))
+                            m.group('value'), filter_spec))
             op = OPERATORS[m.group('op')]
 
         if not m:
@@ -887,85 +889,201 @@ class YoutubeDL(object):
                 '=': operator.eq,
                 '!=': operator.ne,
             }
-            str_operator_rex = re.compile(r'''(?x)\s*\[
+            str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9_-]+)
-                \s*\]$
+                \s*$
                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
-            m = str_operator_rex.search(format_spec)
+            m = str_operator_rex.search(filter_spec)
             if m:
                 comparison_value = m.group('value')
                 op = STR_OPERATORS[m.group('op')]
 
         if not m:
-            raise ValueError('Invalid format specification %r' % format_spec)
+            raise ValueError('Invalid filter specification %r' % filter_spec)
 
         def _filter(f):
             actual_value = f.get(m.group('key'))
             if actual_value is None:
                 return m.group('none_inclusive')
             return op(actual_value, comparison_value)
-        new_formats = [f for f in available_formats if _filter(f)]
+        return _filter
 
-        new_format_spec = format_spec[:-len(m.group(0))]
-        if not new_format_spec:
-            new_format_spec = 'best'
+    def build_format_selector(self, format_spec):
+        def syntax_error(note, start):
+            message = (
+                'Invalid format specification: '
+                '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
+            return SyntaxError(message)
 
-        return (new_format_spec, new_formats)
+        PICKFIRST = 'PICKFIRST'
+        MERGE = 'MERGE'
+        SINGLE = 'SINGLE'
+        FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 
-    def select_format(self, format_spec, available_formats):
-        while format_spec.endswith(']'):
-            format_spec, available_formats = self._apply_format_filter(
-                format_spec, available_formats)
-        if not available_formats:
-            return None
+        def _parse_filter(tokens):
+            filter_parts = []
+            for type, string, start, _, _ in tokens:
+                if type == tokenize.OP and string == ']':
+                    return ''.join(filter_parts)
+                else:
+                    filter_parts.append(string)
 
-        if format_spec in ['best', 'worst', None]:
-            format_idx = 0 if format_spec == 'worst' else -1
-            audiovideo_formats = [
-                f for f in available_formats
-                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
-            if audiovideo_formats:
-                return audiovideo_formats[format_idx]
-            # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
-            elif (all(f.get('acodec') != 'none' for f in available_formats) or
-                  all(f.get('vcodec') != 'none' for f in available_formats)):
-                return available_formats[format_idx]
-        elif format_spec == 'bestaudio':
-            audio_formats = [
-                f for f in available_formats
-                if f.get('vcodec') == 'none']
-            if audio_formats:
-                return audio_formats[-1]
-        elif format_spec == 'worstaudio':
-            audio_formats = [
-                f for f in available_formats
-                if f.get('vcodec') == 'none']
-            if audio_formats:
-                return audio_formats[0]
-        elif format_spec == 'bestvideo':
-            video_formats = [
-                f for f in available_formats
-                if f.get('acodec') == 'none']
-            if video_formats:
-                return video_formats[-1]
-        elif format_spec == 'worstvideo':
-            video_formats = [
-                f for f in available_formats
-                if f.get('acodec') == 'none']
-            if video_formats:
-                return video_formats[0]
-        else:
-            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
-            if format_spec in extensions:
-                filter_f = lambda f: f['ext'] == format_spec
-            else:
-                filter_f = lambda f: f['format_id'] == format_spec
-            matches = list(filter(filter_f, available_formats))
-            if matches:
-                return matches[-1]
-        return None
+        def _parse_format_selection(tokens, endwith=[]):
+            selectors = []
+            current_selector = None
+            for type, string, start, _, _ in tokens:
+                # ENCODING is only defined in python 3.x
+                if type == getattr(tokenize, 'ENCODING', None):
+                    continue
+                elif type in [tokenize.NAME, tokenize.NUMBER]:
+                    current_selector = FormatSelector(SINGLE, string, [])
+                elif type == tokenize.OP:
+                    if string in endwith:
+                        break
+                    if string == ',':
+                        selectors.append(current_selector)
+                        current_selector = None
+                    elif string == '/':
+                        first_choice = current_selector
+                        second_choice = _parse_format_selection(tokens, [','])
+                        current_selector = None
+                        selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
+                    elif string == '[':
+                        if not current_selector:
+                            current_selector = FormatSelector(SINGLE, 'best', [])
+                        format_filter = _parse_filter(tokens)
+                        current_selector.filters.append(format_filter)
+                    elif string == '+':
+                        video_selector = current_selector
+                        audio_selector = _parse_format_selection(tokens, [','])
+                        current_selector = None
+                        selectors.append(FormatSelector(MERGE, (video_selector, audio_selector), []))
+                    else:
+                        raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
+                elif type == tokenize.ENDMARKER:
+                    break
+            if current_selector:
+                selectors.append(current_selector)
+            return selectors
+
+        def _build_selector_function(selector):
+            if isinstance(selector, list):
+                fs = [_build_selector_function(s) for s in selector]
+
+                def selector_function(formats):
+                    for f in fs:
+                        for format in f(formats):
+                            yield format
+                return selector_function
+            elif selector.type == PICKFIRST:
+                fs = [_build_selector_function(s) for s in selector.selector]
+
+                def selector_function(formats):
+                    for f in fs:
+                        picked_formats = list(f(formats))
+                        if picked_formats:
+                            return picked_formats
+                    return []
+            elif selector.type == SINGLE:
+                format_spec = selector.selector
+
+                def selector_function(formats):
+                    if format_spec in ['best', 'worst', None]:
+                        format_idx = 0 if format_spec == 'worst' else -1
+                        audiovideo_formats = [
+                            f for f in formats
+                            if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+                        if audiovideo_formats:
+                            yield audiovideo_formats[format_idx]
+                        # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
+                        elif (all(f.get('acodec') != 'none' for f in formats) or
+                              all(f.get('vcodec') != 'none' for f in formats)):
+                            yield formats[format_idx]
+                    elif format_spec == 'bestaudio':
+                        audio_formats = [
+                            f for f in formats
+                            if f.get('vcodec') == 'none']
+                        if audio_formats:
+                            yield audio_formats[-1]
+                    elif format_spec == 'worstaudio':
+                        audio_formats = [
+                            f for f in formats
+                            if f.get('vcodec') == 'none']
+                        if audio_formats:
+                            yield audio_formats[0]
+                    elif format_spec == 'bestvideo':
+                        video_formats = [
+                            f for f in formats
+                            if f.get('acodec') == 'none']
+                        if video_formats:
+                            yield video_formats[-1]
+                    elif format_spec == 'worstvideo':
+                        video_formats = [
+                            f for f in formats
+                            if f.get('acodec') == 'none']
+                        if video_formats:
+                            yield video_formats[0]
+                    else:
+                        extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
+                        if format_spec in extensions:
+                            filter_f = lambda f: f['ext'] == format_spec
+                        else:
+                            filter_f = lambda f: f['format_id'] == format_spec
+                        matches = list(filter(filter_f, formats))
+                        if matches:
+                            yield matches[-1]
+            elif selector.type == MERGE:
+                def _merge(formats_info):
+                    format_1, format_2 = [f['format_id'] for f in formats_info]
+                    # The first format must contain the video and the
+                    # second the audio
+                    if formats_info[0].get('vcodec') == 'none':
+                        self.report_error('The first format must '
+                                          'contain the video, try using '
+                                          '"-f %s+%s"' % (format_2, format_1))
+                        return
+                    output_ext = (
+                        formats_info[0]['ext']
+                        if self.params.get('merge_output_format') is None
+                        else self.params['merge_output_format'])
+                    return {
+                        'requested_formats': formats_info,
+                        'format': '%s+%s' % (formats_info[0].get('format'),
+                                             formats_info[1].get('format')),
+                        'format_id': '%s+%s' % (formats_info[0].get('format_id'),
+                                                formats_info[1].get('format_id')),
+                        'width': formats_info[0].get('width'),
+                        'height': formats_info[0].get('height'),
+                        'resolution': formats_info[0].get('resolution'),
+                        'fps': formats_info[0].get('fps'),
+                        'vcodec': formats_info[0].get('vcodec'),
+                        'vbr': formats_info[0].get('vbr'),
+                        'stretched_ratio': formats_info[0].get('stretched_ratio'),
+                        'acodec': formats_info[1].get('acodec'),
+                        'abr': formats_info[1].get('abr'),
+                        'ext': output_ext,
+                    }
+                video_selector, audio_selector = map(_build_selector_function, selector.selector)
+
+                def selector_function(formats):
+                    formats = list(formats)
+                    for pair in itertools.product(video_selector(formats), audio_selector(formats)):
+                        yield _merge(pair)
+
+            filters = [self._build_format_filter(f) for f in selector.filters]
+
+            def final_selector(formats):
+                for _filter in filters:
+                    formats = list(filter(_filter, formats))
+                return selector_function(formats)
+            return final_selector
+
+        stream = io.BytesIO(format_spec.encode('utf-8'))
+        tokens = compat_tokenize_tokenize(stream.readline)
+        parsed_selector = _parse_format_selection(tokens)
+        return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):
         res = std_headers.copy()
@@ -1112,52 +1230,8 @@ class YoutubeDL(object):
         if req_format == 'all':
             formats_to_download = formats
         else:
-            for rfstr in req_format.split(','):
-                # We can accept formats requested in the format: 34/5/best, we pick
-                # the first that is available, starting from left
-                req_formats = rfstr.split('/')
-                for rf in req_formats:
-                    if re.match(r'.+?\+.+?', rf) is not None:
-                        # Two formats have been requested like '137+139'
-                        format_1, format_2 = rf.split('+')
-                        formats_info = (self.select_format(format_1, formats),
-                                        self.select_format(format_2, formats))
-                        if all(formats_info):
-                            # The first format must contain the video and the
-                            # second the audio
-                            if formats_info[0].get('vcodec') == 'none':
-                                self.report_error('The first format must '
-                                                  'contain the video, try using '
-                                                  '"-f %s+%s"' % (format_2, format_1))
-                                return
-                            output_ext = (
-                                formats_info[0]['ext']
-                                if self.params.get('merge_output_format') is None
-                                else self.params['merge_output_format'])
-                            selected_format = {
-                                'requested_formats': formats_info,
-                                'format': '%s+%s' % (formats_info[0].get('format'),
-                                                     formats_info[1].get('format')),
-                                'format_id': '%s+%s' % (formats_info[0].get('format_id'),
-                                                        formats_info[1].get('format_id')),
-                                'width': formats_info[0].get('width'),
-                                'height': formats_info[0].get('height'),
-                                'resolution': formats_info[0].get('resolution'),
-                                'fps': formats_info[0].get('fps'),
-                                'vcodec': formats_info[0].get('vcodec'),
-                                'vbr': formats_info[0].get('vbr'),
-                                'stretched_ratio': formats_info[0].get('stretched_ratio'),
-                                'acodec': formats_info[1].get('acodec'),
-                                'abr': formats_info[1].get('abr'),
-                                'ext': output_ext,
-                            }
-                        else:
-                            selected_format = None
-                    else:
-                        selected_format = self.select_format(rf, formats)
-                    if selected_format is not None:
-                        formats_to_download.append(selected_format)
-                        break
+            format_selector = self.build_format_selector(req_format)
+            formats_to_download = list(format_selector(formats))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f9529210d..bc218dd71 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -388,6 +388,10 @@ else:
             pass
         return _terminal_size(columns, lines)
 
+if sys.version_info >= (3, 0):
+    from tokenize import tokenize as compat_tokenize_tokenize
+else:
+    from tokenize import generate_tokens as compat_tokenize_tokenize
 
 __all__ = [
     'compat_HTTPError',
@@ -408,6 +412,7 @@ __all__ = [
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',
+    'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',

From 5acfa126c812c3ab7088af6c7df79697baee7831 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Jun 2015 22:48:02 +0200
Subject: [PATCH 1050/1515] [YoutubeDL] format spec: treat 'all' like a normal
 specifier

So you can use filters with it, for example 'all[width>=400][width<=600]'.
---
 test/test_YoutubeDL.py  |  5 +++++
 youtube_dl/YoutubeDL.py | 13 ++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8f7aef512..709e3100f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -317,6 +317,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+        ydl = YDL({'format': 'all[width>=400][width<=600]'})
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 17a5407b9..258e612af 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -990,7 +990,10 @@ class YoutubeDL(object):
                 format_spec = selector.selector
 
                 def selector_function(formats):
-                    if format_spec in ['best', 'worst', None]:
+                    if format_spec == 'all':
+                        for f in formats:
+                            yield f
+                    elif format_spec in ['best', 'worst', None]:
                         format_idx = 0 if format_spec == 'worst' else -1
                         audiovideo_formats = [
                             f for f in formats
@@ -1226,12 +1229,8 @@ class YoutubeDL(object):
                     req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
-        formats_to_download = []
-        if req_format == 'all':
-            formats_to_download = formats
-        else:
-            format_selector = self.build_format_selector(req_format)
-            formats_to_download = list(format_selector(formats))
+        format_selector = self.build_format_selector(req_format)
+        formats_to_download = list(format_selector(formats))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)

From c4bd188da46a837ddf8f8f8d4766eb799fa2b484 Mon Sep 17 00:00:00 2001
From: Anders Einar Hilden <hildenae@gmail.com>
Date: Mon, 29 Jun 2015 00:11:31 +0200
Subject: [PATCH 1051/1515] NRK now supports / requires HTTPS

Add s? to regexp to support new urls. Update testcases to use HTTPS.
---
 youtube_dl/extractor/nrk.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index cc70c2950..9e4581cf9 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class NRKIE(InfoExtractor):
-    _VALID_URL = r'(?:nrk:|http://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
+    _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
 
     _TESTS = [
         {
@@ -76,7 +76,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
@@ -116,11 +116,11 @@ class NRKPlaylistIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    _VALID_URL = r'(?P<baseurl>https?://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
-            'url': 'http://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
+            'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
             'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
@@ -132,7 +132,7 @@ class NRKTVIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://tv.nrk.no/program/mdfp15000514',
+            'url': 'https://tv.nrk.no/program/mdfp15000514',
             'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
@@ -145,7 +145,7 @@ class NRKTVIE(InfoExtractor):
         },
         {
             # single playlist video
-            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
+            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
             'md5': 'adbd1dbd813edaf532b0a253780719c2',
             'info_dict': {
                 'id': 'MSPO40010515-part2',
@@ -157,7 +157,7 @@ class NRKTVIE(InfoExtractor):
             'skip': 'Only works from Norway',
         },
         {
-            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
+            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
             'playlist': [
                 {
                     'md5': '9480285eff92d64f06e02a5367970a7a',

From bea41c7f3fa4f9072ad2f5354938ab1c8cef0a6d Mon Sep 17 00:00:00 2001
From: corone17 <reino@degeelebosch.nl>
Date: Mon, 29 Jun 2015 00:59:18 +0200
Subject: [PATCH 1052/1515] Update rtlnl.py

Better to extract 'http://manifest.us.rtl.nl' from the json, I'd say. And I think it's better to use the default json-url to make it more futureproof.
Succesfully tested with tarball.
---
 youtube_dl/extractor/rtlnl.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 41d202c28..e708e0093 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -51,7 +51,7 @@ class RtlNlIE(InfoExtractor):
     def _real_extract(self, url):
         uuid = self._match_id(url)
         info = self._download_json(
-            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=adaptive/' % uuid,
             uuid)
 
         material = info['material'][0]
@@ -60,8 +60,8 @@ class RtlNlIE(InfoExtractor):
         description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        videopath = material['videopath'].replace('.f4m', '.m3u8')
-        m3u8_url = 'http://manifest.us.rtl.nl' + videopath
+        videopath = material['videopath'].replace('adaptive', 'flash')
+        m3u8_url = info['meta']['videohost'] + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 

From 738b92632296a9fee3eb7e0c915f6ea6b395125f Mon Sep 17 00:00:00 2001
From: nawl <dylanlwrigh@gmail.com>
Date: Sun, 28 Jun 2015 17:24:00 -0600
Subject: [PATCH 1053/1515] [hentaistigma] Fix video extractor

---
 youtube_dl/extractor/hentaistigma.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index 63d87b74c..225af8cb3 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -32,7 +32,7 @@ class HentaiStigmaIE(InfoExtractor):
         wrap_webpage = self._download_webpage(wrap_url, video_id)
 
         video_url = self._html_search_regex(
-            r'clip:\s*{\s*url: "([^"]*)"', wrap_webpage, 'video url')
+            r'file:"([^"]+)"', wrap_webpage, 'video url')
 
         return {
             'id': video_id,

From 0130afb76e5cb6f470f39f127c8d09eea3e82d0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 29 Jun 2015 12:42:02 +0200
Subject: [PATCH 1054/1515] [YoutubeDL] format spec: allow grouping specifiers
 with parentheses

---
 test/test_YoutubeDL.py  | 24 ++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 39 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 709e3100f..6f374d7ea 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -245,6 +245,30 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], '137+141')
         self.assertEqual(downloaded['ext'], 'mp4')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['137+141', '248+141'])
+
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['136+141', '247+141'])
+
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['248+141'])
+
         for f1, f2 in zip(formats_order, formats_order[1:]):
             info_dict = _make_result([f1, f2], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 258e612af..e5b46f87e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -920,6 +920,7 @@ class YoutubeDL(object):
         PICKFIRST = 'PICKFIRST'
         MERGE = 'MERGE'
         SINGLE = 'SINGLE'
+        GROUP = 'GROUP'
         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 
         def _parse_filter(tokens):
@@ -942,6 +943,10 @@ class YoutubeDL(object):
                 elif type == tokenize.OP:
                     if string in endwith:
                         break
+                    elif string == ')':
+                        # ')' will be handled by the parentheses group
+                        tokens.restore_last_token()
+                        break
                     if string == ',':
                         selectors.append(current_selector)
                         current_selector = None
@@ -955,6 +960,10 @@ class YoutubeDL(object):
                             current_selector = FormatSelector(SINGLE, 'best', [])
                         format_filter = _parse_filter(tokens)
                         current_selector.filters.append(format_filter)
+                    elif string == '(':
+                        if current_selector:
+                            raise syntax_error('Unexpected "("', start)
+                        current_selector = FormatSelector(GROUP, _parse_format_selection(tokens, [')']), [])
                     elif string == '+':
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, [','])
@@ -977,6 +986,8 @@ class YoutubeDL(object):
                         for format in f(formats):
                             yield format
                 return selector_function
+            elif selector.type == GROUP:
+                selector_function = _build_selector_function(selector.selector)
             elif selector.type == PICKFIRST:
                 fs = [_build_selector_function(s) for s in selector.selector]
 
@@ -1084,8 +1095,32 @@ class YoutubeDL(object):
             return final_selector
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
-        tokens = compat_tokenize_tokenize(stream.readline)
-        parsed_selector = _parse_format_selection(tokens)
+        try:
+            tokens = list(compat_tokenize_tokenize(stream.readline))
+        except tokenize.TokenError:
+            raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
+
+        class TokenIterator(object):
+            def __init__(self, tokens):
+                self.tokens = tokens
+                self.counter = 0
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                if self.counter >= len(self.tokens):
+                    raise StopIteration()
+                value = self.tokens[self.counter]
+                self.counter += 1
+                return value
+
+            next = __next__
+
+            def restore_last_token(self):
+                self.counter -= 1
+
+        parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
         return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):

From cf386750c9194839e419a0412f45f25f28236c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 22:21:09 +0600
Subject: [PATCH 1055/1515] [hentaistigma] Modernize

---
 youtube_dl/extractor/hentaistigma.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index 225af8cb3..f5aa73d18 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -19,20 +17,19 @@ class HentaiStigmaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-            r'<h2 class="posttitle"><a[^>]*>([^<]+)</a>',
+            r'<h2[^>]+class="posttitle"[^>]*><a[^>]*>([^<]+)</a>',
             webpage, 'title')
         wrap_url = self._html_search_regex(
-            r'<iframe src="([^"]+mp4)"', webpage, 'wrapper url')
+            r'<iframe[^>]+src="([^"]+mp4)"', webpage, 'wrapper url')
         wrap_webpage = self._download_webpage(wrap_url, video_id)
 
         video_url = self._html_search_regex(
-            r'file:"([^"]+)"', wrap_webpage, 'video url')
+            r'file\s*:\s*"([^"]+)"', wrap_webpage, 'video url')
 
         return {
             'id': video_id,

From f0714c9f869a8b4c2032566af12c107491472361 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Jun 2015 01:02:48 +0600
Subject: [PATCH 1056/1515] [youtube] Speed up upload date regex (#6125)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c28ca9319..ed382e10b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1003,7 +1003,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
         if mobj is None:
             mobj = re.search(
-                r'(?s)id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live) on (.*?)</strong>',
+                r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.*?)</strong>',
                 video_webpage)
         if mobj is not None:
             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())

From c05724cb18eab295b4917a7e13f845e55cac56d9 Mon Sep 17 00:00:00 2001
From: alarig <alarig@swordarmor.fr>
Date: Tue, 30 Jun 2015 16:47:14 +0200
Subject: [PATCH 1057/1515] Add support of HTTPS for ina.fr

---
 youtube_dl/extractor/ina.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ina.py b/youtube_dl/extractor/ina.py
index 0847074ee..65712abc2 100644
--- a/youtube_dl/extractor/ina.py
+++ b/youtube_dl/extractor/ina.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class InaIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
     _TEST = {
         'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
         'md5': 'a667021bf2b41f8dc6049479d9bb38a3',

From 9d0b581feaa3adc72ed3a53b7e8c86fadfddb6e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Jun 2015 20:52:26 +0600
Subject: [PATCH 1058/1515] [youtube] Prefer meta for upload date and modernize

---
 youtube_dl/extractor/youtube.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ed382e10b..6769a009d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -999,15 +999,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
 
         # upload date
-        upload_date = None
-        mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
-        if mobj is None:
-            mobj = re.search(
-                r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.*?)</strong>',
-                video_webpage)
-        if mobj is not None:
-            upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
-            upload_date = unified_strdate(upload_date)
+        upload_date = self._html_search_meta(
+            'datePublished', video_webpage, 'upload date', default=None)
+        if not upload_date:
+            upload_date = self._search_regex(
+                [r'(?s)id="eow-date.*?>(.*?)</span>',
+                 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
+                video_webpage, 'upload date', default=None)
+            if upload_date:
+                upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
+        upload_date = unified_strdate(upload_date)
 
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',

From cf2ac6df6896dac4d23918867bb86fac1e1088d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 30 Jun 2015 19:45:42 +0200
Subject: [PATCH 1059/1515] [YoutubeDL] format spec: Fix handling of '+' with
 '/'

'bestvideo+bestaudio/best' was incorrectly interpreted as 'bestvideo+(bestaudio/best)', so it would fail if 'bestaudio' doesn't exist instead of falling back to 'best'.
---
 test/test_YoutubeDL.py  |  8 ++++++++
 youtube_dl/YoutubeDL.py | 25 +++++++++++++++----------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 6f374d7ea..1e4aaa559 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -245,6 +245,14 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], '137+141')
         self.assertEqual(downloaded['ext'], 'mp4')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '38')
+
         info_dict = _make_result(list(formats_order), extractor='youtube')
         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
         yie = YoutubeIE(ydl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5b46f87e..5deb4848e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -931,7 +931,7 @@ class YoutubeDL(object):
                 else:
                     filter_parts.append(string)
 
-        def _parse_format_selection(tokens, endwith=[]):
+        def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
             for type, string, start, _, _ in tokens:
@@ -941,18 +941,23 @@ class YoutubeDL(object):
                 elif type in [tokenize.NAME, tokenize.NUMBER]:
                     current_selector = FormatSelector(SINGLE, string, [])
                 elif type == tokenize.OP:
-                    if string in endwith:
+                    if string == ')':
+                        if not inside_group:
+                            # ')' will be handled by the parentheses group
+                            tokens.restore_last_token()
                         break
-                    elif string == ')':
-                        # ')' will be handled by the parentheses group
+                    elif inside_merge and string in ['/', ',']:
                         tokens.restore_last_token()
                         break
-                    if string == ',':
+                    elif inside_choice and string == ',':
+                        tokens.restore_last_token()
+                        break
+                    elif string == ',':
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
                         first_choice = current_selector
-                        second_choice = _parse_format_selection(tokens, [','])
+                        second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = None
                         selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
                     elif string == '[':
@@ -963,12 +968,12 @@ class YoutubeDL(object):
                     elif string == '(':
                         if current_selector:
                             raise syntax_error('Unexpected "("', start)
-                        current_selector = FormatSelector(GROUP, _parse_format_selection(tokens, [')']), [])
+                        group = _parse_format_selection(tokens, inside_group=True)
+                        current_selector = FormatSelector(GROUP, group, [])
                     elif string == '+':
                         video_selector = current_selector
-                        audio_selector = _parse_format_selection(tokens, [','])
-                        current_selector = None
-                        selectors.append(FormatSelector(MERGE, (video_selector, audio_selector), []))
+                        audio_selector = _parse_format_selection(tokens, inside_merge=True)
+                        current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                     else:
                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
                 elif type == tokenize.ENDMARKER:

From 1866432db74946c2b66263d38ed2c9d9d7e3177d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 30 Jun 2015 16:22:09 -0300
Subject: [PATCH 1060/1515] Rename --pp-params to --postprocessor-args and
 access value as super class attribute

---
 README.md                          |  2 +-
 youtube_dl/YoutubeDL.py            |  2 +-
 youtube_dl/__init__.py             |  6 +-----
 youtube_dl/options.py              |  4 ++--
 youtube_dl/postprocessor/common.py |  6 ++++--
 youtube_dl/postprocessor/ffmpeg.py | 11 +++++------
 6 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 813ac4a15..7eb17a163 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --pp-params                      Extra parameters for video post-processor.
+    --postprocessor-args             Extra parameters for video post-processor.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3bfe30c76..ff95add78 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -261,7 +261,7 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    pp_params:         Extra parameters for external apps, like avconv.
+    postprocessor_args: Extra parameters for external apps, like avconv.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 8b54d4ae2..356697015 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -171,10 +171,6 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
-    if opts.pp_params is None:
-        opts.pp_params = []
-    else:
-        opts.pp_params = shlex.split(opts.pp_params)
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
@@ -231,7 +227,7 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
-            'extra_params': opts.pp_params
+            'extra_cmd_args': opts.postprocessor_args,
         })
     if opts.convertsubtitles:
         postprocessors.append({
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index fbba9b9d8..3d88428c4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -688,8 +688,8 @@ def parseOpts(overrideArguments=None):
         metavar='FORMAT', dest='recodevideo', default=None,
         help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
     postproc.add_option(
-        '--pp-params',
-        dest='pp_params', default=None, metavar='ARGS',
+        '--postprocessor-args',
+        dest='postprocessor_args', default=None, metavar='ARGS',
         help='Extra parameters for video post-processor.')
     postproc.add_option(
         '-k', '--keep-video',
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index d944d9367..c44501b59 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import os
+import shlex
 
 from ..utils import (
     PostProcessingError,
@@ -23,12 +24,13 @@ class PostProcessor(object):
 
     PostProcessor objects follow a "mutual registration" process similar
     to InfoExtractor objects. And it can receive parameters from CLI trough
-    --pp-params.
+    --postprocessor-args.
     """
 
     _downloader = None
 
-    def __init__(self, downloader=None):
+    def __init__(self, downloader=None, extra_cmd_args=None):
+        self._extra_cmd_args = shlex.split(extra_cmd_args or '')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a696b12b4..891c72769 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -29,8 +29,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None):
-        PostProcessor.__init__(self, downloader)
+    def __init__(self, downloader=None, extra_cmd_args=None):
+        PostProcessor.__init__(self, downloader, extra_cmd_args)
         self._determine_executables()
 
     def check_version(self):
@@ -287,16 +287,15 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None, extra_params=[]):
-        super(FFmpegVideoConvertorPP, self).__init__(downloader)
+    def __init__(self, downloader=None, preferedformat=None, extra_cmd_args=None):
+        super(FFmpegVideoConvertorPP, self).__init__(downloader, extra_cmd_args)
         self._preferedformat = preferedformat
-        self._extra_params = extra_params
 
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
         ext = self._preferedformat
-        options = self._extra_params
+        options = self._extra_cmd_args
         if self._preferedformat == 'xvid':
             ext = 'avi'
             options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])

From 86511ea417ac22e02800fdc3efda1d73a87eb772 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Jul 2015 21:47:56 +0600
Subject: [PATCH 1061/1515] [drtuber] Fix extraction

---
 youtube_dl/extractor/drtuber.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 37c5c181f..639f9182c 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -36,25 +36,24 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            [r'class="hd_title" style="[^"]+">([^<]+)</h1>', r'<title>([^<]+) - \d+'],
+            [r'<p[^>]+class="title_substrate">([^<]+)</p>', r'<title>([^<]+) - \d+'],
             webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
-        like_count = str_to_int(self._html_search_regex(
-            r'<span id="rate_likes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
-            webpage, 'like count', fatal=False))
-        dislike_count = str_to_int(self._html_search_regex(
-            r'<span id="rate_dislikes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
-            webpage, 'like count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r'<span class="comments_count">([\d,\.]+)</span>',
-            webpage, 'comment count', fatal=False))
+        def extract_count(id_, name):
+            return str_to_int(self._html_search_regex(
+                r'<span[^>]+(?:class|id)="%s"[^>]*>([\d,\.]+)</span>' % id_,
+                webpage, '%s count' % name, fatal=False))
+
+        like_count = extract_count('rate_likes', 'like')
+        dislike_count = extract_count('rate_dislikes', 'dislike')
+        comment_count = extract_count('comments_count', 'comment')
 
         cats_str = self._search_regex(
-            r'<span>Categories:</span><div>(.+?)</div>', webpage, 'categories', fatal=False)
+            r'<div[^>]+class="categories_list">(.+?)</div>', webpage, 'categories', fatal=False)
         categories = [] if not cats_str else re.findall(r'<a title="([^"]+)"', cats_str)
 
         return {

From aa5d9a79d6b5c354ee4a6bfbb43f94c2485ab9b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Wed, 1 Jul 2015 20:12:26 -0300
Subject: [PATCH 1062/1515] Simplify `postprocessor_args` transmission to PP
 base class

* Remove `extra_cmd_args` transmission from sub to super class.
* Simplify params transmission through `downloader.params`.
---
 youtube_dl/__init__.py             | 2 +-
 youtube_dl/postprocessor/common.py | 5 ++---
 youtube_dl/postprocessor/ffmpeg.py | 8 ++++----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 356697015..249f76365 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -227,7 +227,6 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
-            'extra_cmd_args': opts.postprocessor_args,
         })
     if opts.convertsubtitles:
         postprocessors.append({
@@ -354,6 +353,7 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
+        'postprocessor_args': shlex.split(opts.postprocessor_args or ''),
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index c44501b59..bee64c457 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import os
-import shlex
 
 from ..utils import (
     PostProcessingError,
@@ -29,8 +28,8 @@ class PostProcessor(object):
 
     _downloader = None
 
-    def __init__(self, downloader=None, extra_cmd_args=None):
-        self._extra_cmd_args = shlex.split(extra_cmd_args or '')
+    def __init__(self, downloader=None):
+        self._extra_cmd_args = downloader.params.get('postprocessor_args')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 891c72769..de8c225da 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -29,8 +29,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None, extra_cmd_args=None):
-        PostProcessor.__init__(self, downloader, extra_cmd_args)
+    def __init__(self, downloader=None):
+        PostProcessor.__init__(self, downloader)
         self._determine_executables()
 
     def check_version(self):
@@ -287,8 +287,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None, extra_cmd_args=None):
-        super(FFmpegVideoConvertorPP, self).__init__(downloader, extra_cmd_args)
+    def __init__(self, downloader=None, preferedformat=None):
+        super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
 
     def run(self, information):

From dbe1a935269fe24fdbb894c325af4b8394c49c04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:05:43 +0800
Subject: [PATCH 1063/1515] [pbs] Fix player URL (closes #6139)

---
 youtube_dl/extractor/pbs.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 143a76696..289f930e0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -90,6 +90,21 @@ class PBSIE(InfoExtractor):
                 'id': 'united-states-of-secrets',
             },
             'playlist_count': 2,
+        },
+        {
+            'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
+            'info_dict': {
+                'id': '2280706814',
+                'display_id': 'player',
+                'ext': 'mp4',
+                'title': 'Death and the Civil War',
+                'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
+                'duration': 6705,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            }
         }
     ]
 
@@ -123,7 +138,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
+                r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 5b61070c700d2460e5aa8ea22f3aa806a2e17dc9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:08:48 +0800
Subject: [PATCH 1064/1515] [pbs] skip_download for m3u8 test cases

---
 youtube_dl/extractor/pbs.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 289f930e0..32c6adeba 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -35,6 +35,9 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
                 'duration': 3190,
             },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -46,6 +49,9 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
                 'duration': 5050,
             },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            }
         },
         {
             'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
@@ -68,7 +74,10 @@ class PBSIE(InfoExtractor):
                 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
-            }
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
@@ -82,7 +91,10 @@ class PBSIE(InfoExtractor):
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
-            }
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
@@ -104,7 +116,7 @@ class PBSIE(InfoExtractor):
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg
-            }
+            },
         }
     ]
 

From 93f942099351895c16ef558366f3eab241cd8793 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:13:27 +0800
Subject: [PATCH 1065/1515] [pbs] Add coding declaration

Python 2.x does not work without it.
---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 32c6adeba..1e2b965f9 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From 8d06a62485ddd2cff396287f6b39614afd1e6033 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 16:47:55 +0800
Subject: [PATCH 1066/1515] [npo] Decrypting token (closes #6136)

---
 youtube_dl/extractor/npo.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 5d8448571..41d7e3fcc 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -16,8 +16,23 @@ class NPOBaseIE(InfoExtractor):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
             video_id, note='Downloading token')
-        return self._search_regex(
+        token = self._search_regex(
             r'npoplayer\.token = "(.+?)"', token_page, 'token')
+        token_l = list(token)
+        first = second = None
+        for i in range(5, len(token_l) - 4):
+            if token_l[i].isdigit():
+                if first is None:
+                    first = i
+                elif second is None:
+                    second = i
+        if first is None or second is None:
+            first = 12
+            second = 13
+
+        token_l[first], token_l[second] = token_l[second], token_l[first]
+
+        return ''.join(token_l)
 
 
 class NPOIE(NPOBaseIE):
@@ -92,7 +107,7 @@ class NPOIE(NPOBaseIE):
 
     def _get_info(self, video_id):
         metadata = self._download_json(
-            'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
+            'http://e.omroep.nl/metadata/%s' % video_id,
             video_id,
             # We have to remove the javascript callback
             transform_source=strip_jsonp,

From f5ca97e393badab4efa07e358b051318955d0b6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jul 2015 20:20:09 +0600
Subject: [PATCH 1067/1515] [npo] Clarify token decryption algorithm source

---
 youtube_dl/extractor/npo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 41d7e3fcc..62d12b7a6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -18,6 +18,7 @@ class NPOBaseIE(InfoExtractor):
             video_id, note='Downloading token')
         token = self._search_regex(
             r'npoplayer\.token = "(.+?)"', token_page, 'token')
+        # Decryption algorithm extracted from http://npoplayer.omroep.nl/csjs/npoplayer-min.js
         token_l = list(token)
         first = second = None
         for i in range(5, len(token_l) - 4):

From 66e568de3b7ed73fd379b941f35ce2b3c7ad9664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jul 2015 21:39:46 +0600
Subject: [PATCH 1068/1515] [extractor/generic] Improve kaltura embeds support
 (Closes #6137)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 32e41d13e..ea60d4a96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -669,6 +669,18 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # Kaltura embed (different embed code)
+        {
+            'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
+            'info_dict': {
+                'id': '1_a52wc67y',
+                'ext': 'flv',
+                'upload_date': '20150127',
+                'uploader_id': 'PremierMedia',
+                'timestamp': int,
+                'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
+            },
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
@@ -1492,8 +1504,8 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = re.search(
-            r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
+        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
+                re.search(r'(?s)(["\'])(?:https?:)?//cdnapisec\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?\1.*?entry_id\s*:\s*(["\'])(?P<id>[^\2]+?)\2', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 

From 91b21b2334a3f8ba54a2b5c5fe4322c46565b9f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 3 Jul 2015 11:53:37 +0800
Subject: [PATCH 1069/1515] [infoq] Fix extraction (closes #6141)

---
 youtube_dl/extractor/infoq.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 117a7faf6..91a1b3ccb 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -5,6 +5,7 @@ import base64
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 
 
@@ -45,7 +46,7 @@ class InfoQIE(InfoExtractor):
         video_id, extension = video_filename.split('.')
 
         http_base = self._search_regex(
-            r'EXPRESSINSTALL_SWF\s*=\s*"(https?://[^/"]+/)', webpage,
+            r'EXPRESSINSTALL_SWF\s*=\s*[^"]*"((?:https?:)?//[^/"]+/)', webpage,
             'HTTP base URL')
 
         formats = [{
@@ -55,7 +56,7 @@ class InfoQIE(InfoExtractor):
             'play_path': playpath,
         }, {
             'format_id': 'http',
-            'url': http_base + real_id,
+            'url': compat_urlparse.urljoin(url, http_base) + real_id,
         }]
         self._sort_formats(formats)
 

From ddf0f74de77cba1d13445996725bfae4a66ac283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jul 2015 22:32:56 +0600
Subject: [PATCH 1070/1515] [howcast] Fix extraction and modernize

---
 youtube_dl/extractor/howcast.py | 35 +++++++++++++++++----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 3f7d6666c..16677f179 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import parse_iso8601
 
 
 class HowcastIE(InfoExtractor):
@@ -13,29 +12,31 @@ class HowcastIE(InfoExtractor):
         'info_dict': {
             'id': '390161',
             'ext': 'mp4',
-            'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.',
             'title': 'How to Tie a Square Knot Properly',
-        }
+            'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
+            'timestamp': 1276081287,
+            'upload_date': '20100609',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id = self._match_id(url)
 
-        video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
-        video_url = self._search_regex(r'\'?file\'?: "(http://mobile-media\.howcast\.com/[0-9]+\.mp4)',
-                                       webpage, 'video URL')
-
-        video_description = self._html_search_regex(r'<meta content=(?:"([^"]+)"|\'([^\']+)\') name=\'description\'',
-                                                    webpage, 'description', fatal=False)
+        embed_code = self._search_regex(
+            r'<iframe[^>]+src="[^"]+\bembed_code=([^\b]+)\b',
+            webpage, 'ooyala embed code')
 
         return {
+            '_type': 'url_transparent',
+            'ie_key': 'Ooyala',
+            'url': 'ooyala:%s' % embed_code,
             'id': video_id,
-            'url': video_url,
-            'title': self._og_search_title(webpage),
-            'description': video_description,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'timestamp': parse_iso8601(self._html_search_meta(
+                'article:published_time', webpage, 'timestamp')),
         }

From 0bcdc276530fe26725857eda5127aefc326ddc47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jul 2015 22:47:52 +0600
Subject: [PATCH 1071/1515] [dailymotion:cloud] Extend _VALID_URL (Closes
 #6145)

---
 youtube_dl/extractor/dailymotion.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 96f0ed9ad..8852f0add 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -254,22 +254,30 @@ class DailymotionUserIE(DailymotionPlaylistIE):
 
 
 class DailymotionCloudIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'http://api\.dmcloud\.net/embed/[^/]+/(?P<id>[^/?]+)'
+    _VALID_URL_PREFIX = r'http://api\.dmcloud\.net/(?:player/)?embed/'
+    _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
+    _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX
 
-    _TEST = {
+    _TESTS = [{
         # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
         # Tested at FranceTvInfo_2
         'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
         'only_matching': True,
-    }
+    }, {
+        # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html
+        'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_dmcloud_url(self, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage)
         if mobj:
             return mobj.group(1)
 
-        mobj = re.search(r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        mobj = re.search(
+            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL,
+            webpage)
         if mobj:
             return mobj.group(1)
 

From 2a46a27e6c704f7b99242243747fa7d8ac68cefc Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 01:07:48 -0700
Subject: [PATCH 1072/1515] [thisamericanlife] Add a new extractor

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/thisamericanlife.py | 32 ++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/thisamericanlife.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d44339200..aba62db53 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -569,6 +569,7 @@ from .tf1 import TF1IE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
+from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
new file mode 100644
index 000000000..6118afa1a
--- /dev/null
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -0,0 +1,32 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ThisAmericanLifeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
+        'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
+        'info_dict': {
+            'id': '487',
+            'title': '487: Harper High School, Part One',
+            'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
+            'ext': 'aac',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
+        media_url = 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': media_url,
+            'ext': 'aac',
+        }

From e41840c522fa09a08812c6e6724063f955426f21 Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 09:35:48 -0700
Subject: [PATCH 1073/1515] [thisamericanlife] get info from <meta> tags

---
 youtube_dl/extractor/thisamericanlife.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 6118afa1a..300884c1d 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 
 class ThisAmericanLifeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
+
     _TEST = {
         'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
         'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
@@ -14,19 +15,26 @@ class ThisAmericanLifeIE(InfoExtractor):
             'title': '487: Harper High School, Part One',
             'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
             'ext': 'aac',
-        }
+            'thumbnail': 'http://www.thisamericanlife.org/sites/default/files/imagecache/large_square/episodes/487_lg_2.jpg',
+            'description': 'We spent five months at Harper High School in Chicago, where last year alone 29 current and recent students were shot. 29. We went to get a sense of what it means to live in the midst of all this gun violence, how teens and adults navigate a world of funerals and Homecoming dances.',
+        },
+        'params': {
+            # m38u download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
-        media_url = 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8'
+        # TODO check to see if there's a free mp3. if so, download that, otherwise get the m3u8 stream.
 
         return {
             'id': video_id,
-            'title': title,
-            'url': media_url,
+            'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),
+            'url': 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8',
             'ext': 'aac',
+            'thumbnail': self._html_search_regex(r'<meta property="og:image" content="(.*?)"', webpage, 'thumbnail'),
+            'description': self._html_search_regex(r'<meta name="description" content="(.*?)"', webpage, 'description'),
         }

From 95506e37af27ee779dd667395f525e4a28455fc2 Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 09:50:39 -0700
Subject: [PATCH 1074/1515] [thisamericanlife] Remove unnecessary comment

---
 youtube_dl/extractor/thisamericanlife.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 300884c1d..295523271 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -28,8 +28,6 @@ class ThisAmericanLifeIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # TODO check to see if there's a free mp3. if so, download that, otherwise get the m3u8 stream.
-
         return {
             'id': video_id,
             'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),

From e56a4c9e9bc459346d23554207b910a1b772000d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jul 2015 05:42:53 +0600
Subject: [PATCH 1075/1515] [thisamericanlife] Improve and simplify

---
 youtube_dl/extractor/thisamericanlife.py | 42 +++++++++++++-----------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 295523271..36493a5de 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -1,38 +1,40 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
 
 class ThisAmericanLifeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
-
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/(?:radio-archives/episode/|play_full\.php\?play=)(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
-        'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
+        'md5': '8f7d2da8926298fdfca2ee37764c11ce',
         'info_dict': {
             'id': '487',
+            'ext': 'm4a',
             'title': '487: Harper High School, Part One',
-            'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
-            'ext': 'aac',
-            'thumbnail': 'http://www.thisamericanlife.org/sites/default/files/imagecache/large_square/episodes/487_lg_2.jpg',
-            'description': 'We spent five months at Harper High School in Chicago, where last year alone 29 current and recent students were shot. 29. We went to get a sense of what it means to live in the midst of all this gun violence, how teens and adults navigate a world of funerals and Homecoming dances.',
+            'description': 'md5:ee40bdf3fb96174a9027f76dbecea655',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # m38u download
-            'skip_download': True,
-        },
-    }
+    }, {
+        'url': 'http://www.thisamericanlife.org/play_full.php?play=487',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(
+            'http://www.thisamericanlife.org/radio-archives/episode/%s' % video_id, video_id)
 
         return {
             'id': video_id,
-            'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),
-            'url': 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8',
-            'ext': 'aac',
-            'thumbnail': self._html_search_regex(r'<meta property="og:image" content="(.*?)"', webpage, 'thumbnail'),
-            'description': self._html_search_regex(r'<meta name="description" content="(.*?)"', webpage, 'description'),
+            'url': 'http://stream.thisamericanlife.org/{0}/stream/{0}_64k.m3u8'.format(video_id),
+            'protocol': 'm3u8_native',
+            'ext': 'm4a',
+            'acodec': 'aac',
+            'vcodec': 'none',
+            'abr': 64,
+            'title': self._html_search_meta(r'twitter:title', webpage, 'title', fatal=True),
+            'description': self._html_search_meta(r'description', webpage, 'description'),
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From 35eb649e9d96027f4d3eec77841791524cc345e2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 4 Jul 2015 09:24:00 +0200
Subject: [PATCH 1076/1515] release 2015.07.04

---
 README.md              | 6 +++---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index e3452c9e1..93e7fb06f 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
@@ -190,8 +190,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
     -F, --list-formats               List all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
+    --youtube-skip-dash-manifest     Do not download the DASH manifests and related data on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv. Ignored if no
                                      merge is required
 
 ## Subtitle Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9a50fbd1c..687936103 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -283,6 +283,7 @@
  - **Motherless**
  - **Motorsport**: motorsport.com
  - **MovieClips**
+ - **MovieFap**
  - **Moviezine**
  - **movshare**: MovShare
  - **MPORA**
@@ -440,6 +441,8 @@
  - **smotri:broadcast**: Smotri.com broadcasts
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
+ - **SnagFilms**
+ - **SnagFilmsEmbed**
  - **Snotr**
  - **Sohu**
  - **soompi**
@@ -502,6 +505,7 @@
  - **TheOnion**
  - **ThePlatform**
  - **TheSixtyOne**
+ - **ThisAmericanLife**
  - **ThisAV**
  - **THVideo**
  - **THVideoPlaylist**
@@ -542,6 +546,7 @@
  - **twitch:stream**
  - **twitch:video**
  - **twitch:vod**
+ - **TwitterCard**
  - **Ubu**
  - **udemy**
  - **udemy:course**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a225e03a1..eff4aebeb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.25'
+__version__ = '2015.07.04'

From d7c9a3e976b00068e926e18d963e9fdb7c3cd678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jul 2015 17:22:11 +0600
Subject: [PATCH 1077/1515] Credit @remitamine for snagfilms (#6096)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 117b9c219..d5418dd37 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -129,3 +129,4 @@ Mister Hat
 Peter Ding
 jackyzy823
 George Brighton
+Remita Amine

From f5f4a27a964b41646303921104f4d6d6fd2098e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Jul 2015 21:30:26 +0200
Subject: [PATCH 1078/1515] [YoutubeDL] format spec: fix handling of '/' with
 ','

When using 'bestvideo/best,bestaudio', 'bestvideo/best' must be set as the current_selector (instead of appending it to the selectors), otherwise when it gets the ',' it would append 'None' to the selectors.
---
 test/test_YoutubeDL.py  | 8 ++++++++
 youtube_dl/YoutubeDL.py | 3 +--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 1e4aaa559..f103779d3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -253,6 +253,14 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], '38')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo/best,bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['137', '141'])
+
         info_dict = _make_result(list(formats_order), extractor='youtube')
         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
         yie = YoutubeIE(ydl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5deb4848e..5a79e5f1d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -958,8 +958,7 @@ class YoutubeDL(object):
                     elif string == '/':
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
-                        current_selector = None
-                        selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
+                        current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
                     elif string == '[':
                         if not current_selector:
                             current_selector = FormatSelector(SINGLE, 'best', [])

From bb8e55366289e0c129ef85abb8c1ac1cbae86a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Jul 2015 21:41:09 +0200
Subject: [PATCH 1079/1515] [YoutubeDL] format spec: Do not fail when a filter
 gives an empty result

For example with 'best[height<40]' we ended getting a 'IndexError: list index out of range'.
---
 test/test_YoutubeDL.py  | 9 ++++++++-
 youtube_dl/YoutubeDL.py | 3 +++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f103779d3..bf2baae07 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -15,7 +15,7 @@ from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_str
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
-from youtube_dl.utils import match_filter_func
+from youtube_dl.utils import ExtractorError, match_filter_func
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -362,6 +362,13 @@ class TestFormatSelection(unittest.TestCase):
         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
         self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
 
+        ydl = YDL({'format': 'best[height<40]'})
+        try:
+            ydl.process_ie_result(info_dict)
+        except ExtractorError:
+            pass
+        self.assertEqual(ydl.downloaded_info_dicts, [])
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a79e5f1d..6478d05dc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1005,6 +1005,9 @@ class YoutubeDL(object):
                 format_spec = selector.selector
 
                 def selector_function(formats):
+                    formats = list(formats)
+                    if not formats:
+                        return
                     if format_spec == 'all':
                         for f in formats:
                             yield f

From ede21449c8d87c2d000d16e5102ee85d87a9b14e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 06:29:36 +0600
Subject: [PATCH 1080/1515] [crunchyroll] Fix extraction (Closes #5855, closes
 #5881)

---
 youtube_dl/extractor/crunchyroll.py | 31 ++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 41f0c736d..73f1e22ef 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -27,7 +27,7 @@ from ..aes import (
 
 
 class CrunchyrollIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
@@ -45,6 +45,22 @@ class CrunchyrollIE(InfoExtractor):
             # rtmp
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
+        'info_dict': {
+            'id': '589804',
+            'ext': 'flv',
+            'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
+            'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Danny Choo Network',
+            'upload_date': '20120213',
+        },
+        'params': {
+            # rtmp
+            'skip_download': True,
+        },
+
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -251,16 +267,17 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
-            # urlencode doesn't work!
-            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' + stream_quality + '&media%5Fid=' + stream_id + '&video%5Fformat=' + stream_format
+            streamdata_req = compat_urllib_request.Request(
+                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
+                % (stream_id, stream_format, stream_quality),
+                compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
             streamdata = self._download_xml(
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
-            video_url = streamdata.find('./host').text
-            video_play_path = streamdata.find('./file').text
+            stream_info = streamdata.find('./{default}preload/stream_info')
+            video_url = stream_info.find('./host').text
+            video_play_path = stream_info.find('./file').text
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From 43232d5c143b1025671f70ff34f5c0f28ec56847 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 19:01:07 +0600
Subject: [PATCH 1081/1515] [rtlnl] Improve

---
 youtube_dl/extractor/rtlnl.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e708e0093..d94861325 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -59,9 +59,11 @@ class RtlNlIE(InfoExtractor):
         subtitle = material['title'] or info['episodes'][0]['name']
         description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
+        meta = info.get('meta', {})
+
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        videopath = material['videopath'].replace('adaptive', 'flash')
-        m3u8_url = info['meta']['videohost'] + videopath
+        videopath = material['videopath'].replace('/adaptive/', '/flash/')
+        m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
@@ -82,7 +84,7 @@ class RtlNlIE(InfoExtractor):
         self._sort_formats(formats)
 
         thumbnails = []
-        meta = info.get('meta', {})
+
         for p in ('poster_base_url', '"thumb_base_url"'):
             if not meta.get(p):
                 continue

From 9dfc4fa1a1a922fb6c0212d09d28b4a8495c1031 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 19:07:07 +0600
Subject: [PATCH 1082/1515] [rtlnl] Add test with encrypted m3u8 streams for
 reference

---
 youtube_dl/extractor/rtlnl.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index d94861325..049deaf8d 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -43,6 +43,10 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        # encrypted m3u8 streams
+        'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
+        'only_matching': True,
     }, {
         'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
         'only_matching': True,

From 89d42c2c758a3cb06b71145c24bfe8057419295a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jul 2015 02:58:02 +0600
Subject: [PATCH 1083/1515] [rtlnl] Clarify test

---
 youtube_dl/extractor/rtlnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 049deaf8d..1228245ad 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -44,7 +44,7 @@ class RtlNlIE(InfoExtractor):
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
     }, {
-        # encrypted m3u8 streams
+        # encrypted m3u8 streams, georestricted
         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
         'only_matching': True,
     }, {

From 0c20ee7d4b0c98221c71769482980a0a218dadb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jul 2015 04:16:56 +0600
Subject: [PATCH 1084/1515] [rtlnl] Clarify current adaptive -> flash
 workaround rationale

---
 youtube_dl/extractor/rtlnl.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 1228245ad..a4d3d73ff 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -66,6 +66,9 @@ class RtlNlIE(InfoExtractor):
         meta = info.get('meta', {})
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
+        # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
+        # this adaptive -> flash workaround is not required in general, but it also
+        # allows bypassing georestriction therefore is retained for now.
         videopath = material['videopath'].replace('/adaptive/', '/flash/')
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 

From 13af92fdc443f753dcb3cb91736b09a4ae85c36a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 08:39:38 +0800
Subject: [PATCH 1085/1515] [common] Add 'fatal' to _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 81623bfe3..d859aea52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,8 @@ class InfoExtractor(object):
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
-                              m3u8_id=None, note=None, errnote=None):
+                              m3u8_id=None, note=None, errnote=None,
+                              fatal=True):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -866,7 +867,10 @@ class InfoExtractor(object):
         m3u8_doc = self._download_webpage(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
-            errnote=errnote or 'Failed to download m3u8 information')
+            errnote=errnote or 'Failed to download m3u8 information',
+            fatal=fatal)
+        if m3u8_doc is False:
+            return m3u8_doc
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 59a83d3e5b9c09d0c6e8fb430ea99d71ef6e2eba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 08:40:38 +0800
Subject: [PATCH 1086/1515] [spiegeltv] Skip invalid m3u8 manifests (closes
 #6157)

---
 youtube_dl/extractor/spiegeltv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 08a5c4314..27f4033c5 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,11 +77,13 @@ class SpiegeltvIE(InfoExtractor):
                     'rtmp_live': True,
                 })
             elif determine_ext(endpoint) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     endpoint.replace('[video]', play_path),
                     video_id, 'm4v',
                     preference=1,  # Prefer hls since it allows to workaround georestriction
-                    m3u8_id='hls'))
+                    m3u8_id='hls', fatal=False)
+                if m3u8_formats is not False:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'url': endpoint,

From 37c1e4025c6df834e93a64c1c13eebac23e90942 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 6 Jul 2015 15:26:49 +0800
Subject: [PATCH 1087/1515] [yinyuetai] New extractor for yinyuetai.com

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/yinyuetai.py | 47 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/yinyuetai.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aba62db53..8665855eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -733,6 +733,7 @@ from .yandexmusic import (
     YandexMusicPlaylistIE,
 )
 from .yesjapan import YesJapanIE
+from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
new file mode 100644
index 000000000..661c34602
--- /dev/null
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class YinYueTaiIE(InfoExtractor):
+    IE_NAME = 'yinyuetai:video'
+    _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://v.yinyuetai.com/video/2322376',
+        'md5': '6e3abe28d38e3a54b591f9f040595ce0',
+        'info_dict': {
+            'id': '2322376',
+            'ext': 'mp4',
+            'title': '少女时代_PARTY_Music Video Teaser',
+            'creator': '少女时代',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://ext.yinyuetai.com/main/get-h-mv-info?json=true&videoId=%s' % video_id, video_id,
+            'Downloading mv info')['videoInfo']['coreVideoInfo']
+
+        if info['error']:
+            raise ExtractorError(info['errorMsg'], expected=True)
+
+        formats = [
+            {'url': format_info['videoUrl'], 'format_id': format_info['qualityLevel'],
+             'format': format_info['qualityLevelName'], 'filesize': format_info['fileSize'],
+             'ext': 'mp4', 'preference': format_info['bitrate']}
+            for format_info in info['videoUrlModels']
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['videoName'],
+            'thumbnail': info['bigHeadImage'],
+            'creator': info['artistNames'],
+            'duration': info['duration'],
+            'formats': formats,
+        }

From e9d33454b552c7a5a2e048f439b90a58aa5528ad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 16:19:49 +0800
Subject: [PATCH 1088/1515] [qqmusic:playlist] Playlist names are optional

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index f9aafcd28..d6bc05b7b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -274,6 +274,6 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
             ) for song in list_json['songlist']
         ]
 
-        list_name = list_json['dissname']
+        list_name = list_json.get('dissname')
         list_description = clean_html(unescapeHTML(list_json.get('desc')))
         return self.playlist_result(entries, list_id, list_name, list_description)

From fc7ae675e26a98bc74918f311d22f515ec7e0477 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:08:32 +0800
Subject: [PATCH 1089/1515] [qqmusic:album] Strip description

---
 youtube_dl/extractor/qqmusic.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e704640e5..6d85d58e3 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -169,7 +169,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
-            'description': 'md5:712f0cdbfc7e776820d08150e6df593d',
+            'description': 'md5:179c5dce203a5931970d306aa9607ea6',
         },
         'playlist_count': 4,
     }, {
@@ -177,7 +177,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '002Y5a3b3AlCu3',
             'title': '그리고...',
-            'description': 'md5:b1d133b8c9bac8fed4e1a97df759f4cf',
+            'description': 'md5:a48823755615508a95080e81b51ba729',
         },
         'playlist_count': 8,
     }]
@@ -196,6 +196,8 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         ]
         album_name = album['name']
         album_detail = album.get('desc')
+        if album_detail is not None:
+            album_detail = album_detail.strip()
 
         return self.playlist_result(entries, mid, album_name, album_detail)
 

From dfc4eca21f7d34b5e65f42b284b24077c8bbc109 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:09:17 +0800
Subject: [PATCH 1090/1515] [qqmusic:album] Playlist names are optional

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 6d85d58e3..03e6c688f 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -194,7 +194,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
                 'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
             ) for song in album['list']
         ]
-        album_name = album['name']
+        album_name = album.get('name')
         album_detail = album.get('desc')
         if album_detail is not None:
             album_detail = album_detail.strip()

From 85a064861f5213abf2b56dd671e1f45188a02adf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:54:41 +0800
Subject: [PATCH 1091/1515] [qqmusic] Use regex for thumbnails in test cases

---
 youtube_dl/extractor/qqmusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 8a724ab51..476432330 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -27,7 +27,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
-            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/7/p/001IV22P1RDX7p.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }, {
         'note': 'There is no mp3-320 version of this song.',
@@ -40,7 +40,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20050626',
             'creator': '李季美',
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
-            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/r/Q/0042owYj46IxrQ.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }]
 

From 275c0423aa85691fc78dda253fa5d00ab471e7b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jul 2015 00:02:34 +0600
Subject: [PATCH 1092/1515] [vk] Fix extraction (Closes #6153)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f2ae109f9..4ae28d4ca 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -153,7 +153,7 @@ class VKIE(InfoExtractor):
         if not video_id:
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
-        info_url = 'http://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+        info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
         ERRORS = {

From 7f220b2facecde21ac3257d20938b63c1b6b01fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jul 2015 00:04:19 +0600
Subject: [PATCH 1093/1515] [vk] Catch ownership confirmation request

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 4ae28d4ca..ccd897084 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -156,6 +156,11 @@ class VKIE(InfoExtractor):
         info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
+        if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
+            raise ExtractorError(
+                'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',
+                expected=True)
+
         ERRORS = {
             r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
             'Video %s has been removed from public access due to rightholder complaint.',

From d7b4d5dd5072fc82a7f3d437e5990aa74a35a100 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Jul 2015 14:16:56 +0800
Subject: [PATCH 1094/1515] [gfycat] Extract id correctly (fixes #6165)

---
 youtube_dl/extractor/gfycat.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 397f1d42e..048ee31e2 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class GfycatIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/)?(?P<id>[^/?#]+)'
+    _TESTS = [{
         'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
         'info_dict': {
             'id': 'DeadlyDecisiveGermanpinscher',
@@ -27,7 +27,23 @@ class GfycatIE(InfoExtractor):
             'categories': list,
             'age_limit': 0,
         }
-    }
+    }, {
+        'url': 'http://gfycat.com/ifr/JauntyTimelyAmazontreeboa',
+        'info_dict': {
+            'id': 'JauntyTimelyAmazontreeboa',
+            'ext': 'mp4',
+            'title': 'JauntyTimelyAmazontreeboa',
+            'timestamp': 1411720126,
+            'upload_date': '20140926',
+            'uploader': 'anonymous',
+            'duration': 3.52,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'categories': list,
+            'age_limit': 0,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 267dc07e6b71277b67abad08c515ca9c1bb09f61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Jul 2015 14:22:13 +0800
Subject: [PATCH 1095/1515] [gfycat] Catch errors

---
 youtube_dl/extractor/gfycat.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 048ee31e2..884700c52 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -6,6 +6,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     qualities,
+    ExtractorError,
 )
 
 
@@ -50,7 +51,10 @@ class GfycatIE(InfoExtractor):
 
         gfy = self._download_json(
             'http://gfycat.com/cajax/get/%s' % video_id,
-            video_id, 'Downloading video info')['gfyItem']
+            video_id, 'Downloading video info')
+        if 'error' in gfy:
+            raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)
+        gfy = gfy['gfyItem']
 
         title = gfy.get('title') or gfy['gfyName']
         description = gfy.get('description')

From 4dd09c9addeaccad6ad6d33f01d6240f3594fc84 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 7 Jul 2015 10:36:07 +0200
Subject: [PATCH 1096/1515] release 2015.07.07

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 687936103..0ca06c71d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -384,6 +384,7 @@
  - **Pyvideo**
  - **qqmusic**
  - **qqmusic:album**
+ - **qqmusic:playlist**
  - **qqmusic:singer**
  - **qqmusic:toplist**
  - **QuickVid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eff4aebeb..3364647ed 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.04'
+__version__ = '2015.07.07'

From cbc1fadd6f544619b711209b68ea15a912ca0fa1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jul 2015 13:40:21 +0800
Subject: [PATCH 1097/1515] [clipsyndicate] Support chic subdomain (fixes
 #6176)

---
 youtube_dl/extractor/clipsyndicate.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index d07d544ea..b1e45a677 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -10,9 +10,9 @@ from ..utils import (
 
 
 class ClipsyndicateIE(InfoExtractor):
-    _VALID_URL = r'http://www\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
         'md5': '4d7d549451bad625e0ff3d7bd56d776c',
         'info_dict': {
@@ -22,7 +22,10 @@ class ClipsyndicateIE(InfoExtractor):
             'duration': 612,
             'thumbnail': 're:^https?://.+\.jpg',
         },
-    }
+    }, {
+        'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 1316b54956ac871b4644b6760c5bf88207b2e58f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jul 2015 13:43:23 +0800
Subject: [PATCH 1098/1515] [clipsyndicate] Use _match_id

---
 youtube_dl/extractor/clipsyndicate.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index b1e45a677..8306d6fb7 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
@@ -28,8 +26,7 @@ class ClipsyndicateIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         js_player = self._download_webpage(
             'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id,
             video_id, 'Downlaoding player')

From cf9cf7dd04d8e15635084dcf9f7a76acff5ac218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:27:06 +0600
Subject: [PATCH 1099/1515] [vk] Extend _VALID_URL to handle biqle.ru (Closes
 #6179)

---
 youtube_dl/extractor/vk.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ccd897084..0f7ce45ca 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -21,7 +21,17 @@ from ..utils import (
 
 class VKIE(InfoExtractor):
     IE_NAME = 'vk.com'
-    _VALID_URL = r'https?://(?:m\.)?vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|(?:.+?\?.*?z=)?video(?P<videoid>[^s].*?)(?:\?|%2F|$))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:m\.)?vk\.com/video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|
+                            (?:
+                                (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
+                                (?:www\.)?biqle\.ru/watch/
+                            )
+                            (?P<videoid>[^s].*?)(?:\?|%2F|$)
+                        )
+                    '''
     _NETRC_MACHINE = 'vk'
 
     _TESTS = [
@@ -114,6 +124,11 @@ class VKIE(InfoExtractor):
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'only_matching': True,
         },
+        {
+            # vk wrapper
+            'url': 'http://www.biqle.ru/watch/847655_160197695',
+            'only_matching': True,
+        }
     ]
 
     def _login(self):

From 4647845679429739f72b0eba5b963e096ec5bf86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:34:50 +0600
Subject: [PATCH 1100/1515] [vk] Fix youtube extraction

---
 youtube_dl/extractor/vk.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0f7ce45ca..697064175 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -195,10 +195,11 @@ class VKIE(InfoExtractor):
             if re.search(error_re, info_page):
                 raise ExtractorError(error_msg % video_id, expected=True)
 
-        m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
-        if m_yt is not None:
-            self.to_screen('Youtube video detected')
-            return self.url_result(m_yt.group(1), 'Youtube')
+        youtube_url = self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+            info_page, 'youtube iframe', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube', video_id)
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)

From 9281f6d25338c179e7bc2f48352da3311102158e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:41:08 +0600
Subject: [PATCH 1101/1515] [vk] Add test for youtube embed

---
 youtube_dl/extractor/vk.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 697064175..23d153031 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -119,6 +119,21 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # youtube embed
+            'url': 'https://vk.com/video276849682_170681728',
+            'info_dict': {
+                'id': 'V3K4mi0SYkc',
+                'ext': 'mp4',
+                'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
+                'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
+                'duration': 179,
+                'upload_date': '20130116',
+                'uploader': "Children's Joy Foundation",
+                'uploader_id': 'thecjf',
+                'view_count': int,
+            },
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -199,7 +214,7 @@ class VKIE(InfoExtractor):
             r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
             info_page, 'youtube iframe', default=None)
         if youtube_url:
-            return self.url_result(youtube_url, 'Youtube', video_id)
+            return self.url_result(youtube_url, 'Youtube')
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)

From e2082ea9422aadf7ae2580d9333008279cda51f0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 00:50:32 +0800
Subject: [PATCH 1102/1515] [yinyuetai] Add test for h5/ part in _VALID_URL

---
 youtube_dl/extractor/yinyuetai.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 661c34602..41ee89da4 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
     _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',
         'md5': '6e3abe28d38e3a54b591f9f040595ce0',
         'info_dict': {
@@ -17,7 +17,10 @@ class YinYueTaiIE(InfoExtractor):
             'title': '少女时代_PARTY_Music Video Teaser',
             'creator': '少女时代',
         },
-    }
+    }, {
+        'url': 'http://v.yinyuetai.com/video/h5/2322376',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From af0f9b0e95233862e758140b282497d04edfb885 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 00:54:37 +0800
Subject: [PATCH 1103/1515] [yinyuetai] Style

---
 youtube_dl/extractor/yinyuetai.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 41ee89da4..a4ada4872 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -32,12 +32,14 @@ class YinYueTaiIE(InfoExtractor):
         if info['error']:
             raise ExtractorError(info['errorMsg'], expected=True)
 
-        formats = [
-            {'url': format_info['videoUrl'], 'format_id': format_info['qualityLevel'],
-             'format': format_info['qualityLevelName'], 'filesize': format_info['fileSize'],
-             'ext': 'mp4', 'preference': format_info['bitrate']}
-            for format_info in info['videoUrlModels']
-        ]
+        formats = [{
+            'url': format_info['videoUrl'],
+            'format_id': format_info['qualityLevel'],
+            'format': format_info['qualityLevelName'],
+            'filesize': format_info['fileSize'],
+            'ext': 'mp4',
+            'preference': format_info['bitrate'],
+        } for format_info in info['videoUrlModels']]
         self._sort_formats(formats)
 
         return {

From d76dea001b5365b7646986cba12f9908cd321f6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 01:07:45 +0800
Subject: [PATCH 1104/1515] [yinyuetai] Miscellaneous improvements

1. Include all fields in _TEST
2. Use .get() for optional fields
3. Clarify the intention of 'ext' in formats
---
 youtube_dl/extractor/yinyuetai.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index a4ada4872..003df9233 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -16,6 +16,8 @@ class YinYueTaiIE(InfoExtractor):
             'ext': 'mp4',
             'title': '少女时代_PARTY_Music Video Teaser',
             'creator': '少女时代',
+            'duration': 25,
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }, {
         'url': 'http://v.yinyuetai.com/video/h5/2322376',
@@ -35,18 +37,19 @@ class YinYueTaiIE(InfoExtractor):
         formats = [{
             'url': format_info['videoUrl'],
             'format_id': format_info['qualityLevel'],
-            'format': format_info['qualityLevelName'],
-            'filesize': format_info['fileSize'],
+            'format': format_info.get('qualityLevelName'),
+            'filesize': format_info.get('fileSize'),
+            # though URLs ends with .flv, the downloaded files are in fact mp4
             'ext': 'mp4',
-            'preference': format_info['bitrate'],
+            'tbr': format_info.get('bitrate'),
         } for format_info in info['videoUrlModels']]
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': info['videoName'],
-            'thumbnail': info['bigHeadImage'],
-            'creator': info['artistNames'],
-            'duration': info['duration'],
+            'thumbnail': info.get('bigHeadImage'),
+            'creator': info.get('artistNames'),
+            'duration': info.get('duration'),
             'formats': formats,
         }

From 082a0140eff05dfdac1020548a49541fcddd6ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 23:40:19 +0600
Subject: [PATCH 1105/1515] [yinyuetai] Do not capture unused group

---
 youtube_dl/extractor/yinyuetai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 003df9233..fa6b40816 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
-    _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',
         'md5': '6e3abe28d38e3a54b591f9f040595ce0',

From 77c6fb5b24c477c9de022f252dc9958a4b3b8b42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 20:48:38 +0600
Subject: [PATCH 1106/1515] [youtube] Make further DASH manifests not fatal
 after succeeded one

---
 youtube_dl/extractor/youtube.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6769a009d..3c629d38a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -798,7 +798,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
     def _parse_dash_manifest(
-            self, video_id, dash_manifest_url, player_url, age_gate):
+            self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
         def decrypt_sig(mobj):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@@ -807,7 +807,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         dash_doc = self._download_xml(
             dash_manifest_url, video_id,
             note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest')
+            errnote='Could not download DASH manifest',
+            fatal=fatal)
+
+        if dash_doc is False:
+            return []
 
         formats = []
         for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
@@ -1161,14 +1165,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
+            dash_mpd_fatal = True
             for dash_manifest_url in dash_mpds:
                 dash_formats = {}
                 try:
                     for df in self._parse_dash_manifest(
-                            video_id, dash_manifest_url, player_url, age_gate):
+                            video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df
+                        # Additional DASH manifests may end up in HTTP Error 403 therefore
+                        # allow them to fail without bug report message if we already have
+                        # some DASH manifest succeeded. This is temporary workaround to reduce
+                        # burst of bug reports until we figure out the reason and whether it
+                        # can be fixed at all.
+                        dash_mpd_fatal = False
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)

From 675e9f22ea3f43fbade9bbd13ee5de3eb45c538f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 23:52:03 +0600
Subject: [PATCH 1107/1515] [vimple] Extract spruto player based extractor
 class

---
 youtube_dl/extractor/vimple.py | 46 +++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index aa3d6ddfd..92321d66e 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -4,7 +4,29 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class VimpleIE(InfoExtractor):
+class SprutoBaseIE(InfoExtractor):
+    def _extract_spruto(self, spruto, video_id):
+        playlist = spruto['playlist'][0]
+        title = playlist['title']
+        video_id = playlist.get('videoId') or video_id
+        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
+        duration = int_or_none(playlist.get('duration'))
+
+        formats = [{
+            'url': f['url'],
+        } for f in playlist['video']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class VimpleIE(SprutoBaseIE):
     IE_DESC = 'Vimple - one-click video hosting'
     _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
     _TESTS = [
@@ -30,25 +52,9 @@ class VimpleIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://player.vimple.ru/iframe/%s' % video_id, video_id)
 
-        playlist = self._parse_json(
+        spruto = self._parse_json(
             self._search_regex(
                 r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
-            video_id)['playlist'][0]
+            video_id)
 
-        title = playlist['title']
-        video_id = playlist.get('videoId') or video_id
-        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
-        duration = int_or_none(playlist.get('duration'))
-
-        formats = [{
-            'url': f['url'],
-        } for f in playlist['video']]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        return self._extract_spruto(spruto, video_id)

From 1c20ddc966a69a241027c2d9a132b9caf3d0ebde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 23:53:50 +0600
Subject: [PATCH 1108/1515] [myvi:embed] Add extractor (Closes #6167)

---
 youtube_dl/extractor/myvi.py | 44 ++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/myvi.py

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
new file mode 100644
index 000000000..9f4330f50
--- /dev/null
+++ b/youtube_dl/extractor/myvi.py
@@ -0,0 +1,44 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .vimple import SprutoBaseIE
+
+
+class MyviEmbedIE(SprutoBaseIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        myvi\.ru/player/
+                            (?:
+                                (?:
+                                    embed/html|
+                                    api/Video/Get
+                                )/|
+                                content/preloader\.swf\?.*\bid=
+                            )
+                            (?P<id>[\da-zA-Z_]+)
+                    '''
+    _TESTS = [{
+        'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
+        'md5': '571bbdfba9f9ed229dc6d34cc0f335bf',
+        'info_dict': {
+            'id': 'f16b2bbd-cde8-481c-a981-7cd48605df43',
+            'ext': 'mp4',
+            'title': 'хозяин жизни',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 25,
+        },
+    }, {
+        'url': 'http://myvi.ru/player/content/preloader.swf?id=oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wOYf1WFpPfc_bWTKGVf_Zafr0',
+        'only_matching': True,
+    }, {
+        'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        spruto = self._download_json(
+            'http://myvi.ru/player/api/Video/Get/%s?sig' % video_id, video_id)['sprutoData']
+
+        return self._extract_spruto(spruto, video_id)

From 83423254ccdf60cb8756aaf7900c929c1cf1a3ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:07:26 +0600
Subject: [PATCH 1109/1515] [myvi:embed] Extend _VALID_URL

---
 youtube_dl/extractor/myvi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index 9f4330f50..a14a5365b 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -7,7 +7,7 @@ from .vimple import SprutoBaseIE
 class MyviEmbedIE(SprutoBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        myvi\.ru/player/
+                        myvi\.(?:ru/player|tv)/
                             (?:
                                 (?:
                                     embed/html|
@@ -33,6 +33,9 @@ class MyviEmbedIE(SprutoBaseIE):
     }, {
         'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
         'only_matching': True,
+    }, {
+        'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e6c2d9ad29bcc4eaa0eed03d3852588b6c7a10c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:25:36 +0600
Subject: [PATCH 1110/1515] [extractor/generic:myvi] Add support for myvi
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 youtube_dl/extractor/myvi.py    | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ea60d4a96..f8d6a8c76 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,6 +37,7 @@ from .rutv import RUTVIE
 from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
+from .myvi import MyviEmbedIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
@@ -1425,6 +1426,11 @@ class GenericIE(InfoExtractor):
         if smotri_url:
             return self.url_result(smotri_url, 'Smotri')
 
+        # Look for embedded Myvi.ru player
+        myvi_url = MyviEmbedIE._extract_url(webpage)
+        if myvi_url:
+            return self.url_result(myvi_url)
+
         # Look for embeded soundcloud player
         mobj = re.search(
             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index a14a5365b..896080c1e 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .vimple import SprutoBaseIE
 
 
@@ -38,6 +40,13 @@ class MyviEmbedIE(SprutoBaseIE):
         'only_matching': True,
     }]
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/embed/html/[^"]+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From f2f89c762a13392e8876c5e9dff8b418c3912ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:27:02 +0600
Subject: [PATCH 1111/1515] [myvi:embed] Improve _VALID_URL

---
 youtube_dl/extractor/myvi.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index 896080c1e..cd80cc177 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -13,11 +13,12 @@ class MyviEmbedIE(SprutoBaseIE):
                             (?:
                                 (?:
                                     embed/html|
+                                    flash|
                                     api/Video/Get
                                 )/|
                                 content/preloader\.swf\?.*\bid=
                             )
-                            (?P<id>[\da-zA-Z_]+)
+                            (?P<id>[\da-zA-Z_-]+)
                     '''
     _TESTS = [{
         'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
@@ -38,12 +39,15 @@ class MyviEmbedIE(SprutoBaseIE):
     }, {
         'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
         'only_matching': True,
+    }, {
+        'url': 'http://myvi.ru/player/flash/ocp2qZrHI-eZnHKQBK4cZV60hslH8LALnk0uBfKsB-Q4WnY26SeGoYPi8HWHxu0O30',
+        'only_matching': True,
     }]
 
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/embed/html/[^"]+)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/(?:embed/html|flash)/[^"]+)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 6dd94d3a79353f8e694efaf2fa27f4bb40227aff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:27:44 +0600
Subject: [PATCH 1112/1515] [myvi:embed] Rename to myvi

---
 youtube_dl/extractor/__init__.py | 1 +
 youtube_dl/extractor/generic.py  | 4 ++--
 youtube_dl/extractor/myvi.py     | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a0e3b333d..a64b457bd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -323,6 +323,7 @@ from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
+from .myvi import MyviIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f8d6a8c76..7c604c554 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,7 +37,7 @@ from .rutv import RUTVIE
 from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
-from .myvi import MyviEmbedIE
+from .myvi import MyviIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
@@ -1427,7 +1427,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(smotri_url, 'Smotri')
 
         # Look for embedded Myvi.ru player
-        myvi_url = MyviEmbedIE._extract_url(webpage)
+        myvi_url = MyviIE._extract_url(webpage)
         if myvi_url:
             return self.url_result(myvi_url)
 
diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index cd80cc177..4c65be122 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -6,7 +6,7 @@ import re
 from .vimple import SprutoBaseIE
 
 
-class MyviEmbedIE(SprutoBaseIE):
+class MyviIE(SprutoBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         myvi\.(?:ru/player|tv)/

From 06a12933f3621c4b5b84346edc0d5bd570f1aac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:58:01 +0600
Subject: [PATCH 1113/1515] [pbs] Add support for subtitles (Closes #6184)

---
 youtube_dl/extractor/pbs.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 1e2b965f9..fec5d65ad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -224,6 +224,14 @@ class PBSIE(InfoExtractor):
             rating_str = rating_str.rpartition('-')[2]
         age_limit = US_RATINGS.get(rating_str)
 
+        subtitles = {}
+        closed_captions_url = info.get('closed_captions_url')
+        if closed_captions_url:
+            subtitles['en'] = [{
+                'ext': 'ttml',
+                'url': closed_captions_url,
+            }]
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -234,4 +242,5 @@ class PBSIE(InfoExtractor):
             'age_limit': age_limit,
             'upload_date': upload_date,
             'formats': formats,
+            'subtitles': subtitles,
         }

From bf20b9c5405b276e6de29b1b28b2e3ad2182e891 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 01:15:55 +0600
Subject: [PATCH 1114/1515] [extractor/generic] Add test for myvi embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7c604c554..392ad3648 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -339,6 +339,17 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # Myvi.ru embed
+        {
+            'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
+            'info_dict': {
+                'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
+                'ext': 'mp4',
+                'title': 'Ужастики, русский трейлер (2015)',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 153,
+            }
+        },
         # XHamster embed
         {
             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',

From 02b386f80abc16f5334e980c2fb729b0cc69a0d6 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 13:29:57 +0800
Subject: [PATCH 1115/1515] [neteasemusic] Changes after review

---
 youtube_dl/extractor/neteasemusic.py | 42 ++++++++++++++++++----------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index a70c65ca5..419f3ecc9 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from hashlib import md5
+from base64 import b64encode
 from datetime import datetime
 import itertools
 import re
@@ -20,14 +21,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(str(cls._NETEASE_SALT))
+        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
         string_bytes = bytearray(str(dfsid))
         salt_len = len(salt_bytes)
-        for i in xrange(len(string_bytes)):
+        for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
         m.update(string_bytes)
-        result = m.digest().encode('base64')[:-1]
+        result = b64encode(m.digest())
         return result.replace('/', '_').replace('+', '-')
 
     @classmethod
@@ -41,12 +42,11 @@ class NetEaseMusicBaseIE(InfoExtractor):
                 'url': 'http://m1.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
-                'ext': details['extension'],
-                'abr': details['bitrate'] / 1000,
-                'preference': details['bitrate'],
+                'ext': details.get('extension'),
+                'abr': details.get('bitrate', 0) / 1000,
                 'format_id': song_format,
-                'filesize': details['size'],
-                'asr': details['sr']
+                'filesize': details.get('size'),
+                'asr': details.get('sr')
             })
         return formats
 
@@ -98,6 +98,19 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20080211',
             'timestamp': 1202745600,
         },
+    }, {
+        'note': 'Has translated name.',
+        'url': 'http://music.163.com/#/song?id=22735043',
+        'info_dict': {
+            'id': '22735043',
+            'ext': 'mp3',
+            'title': '소원을 말해봐 (Genie)',
+            'creator': '少女时代',
+            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
+            'upload_date': '20100127',
+            'timestamp': 1264608000,
+            'alt_title': '说出愿望吧(Genie)',
+        }
     }]
 
     def _process_lyrics(self, lyrics_info):
@@ -109,10 +122,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
         lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
         original_ts_texts = re.findall(lyrics_expr, original)
-        translation_ts_dict = {
-            time_stamp: text for time_stamp, text in re.findall(lyrics_expr, translated)
-        }
-
+        translation_ts_dict = dict(
+            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
+        )
         lyrics = '\n'.join([
             '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
             for time_stamp, text in original_ts_texts
@@ -139,8 +151,8 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
         lyrics = self._process_lyrics(lyrics_info)
 
         alt_title = None
-        if info.get('alias'):
-            alt_title = '/'.join(info.get('alias'))
+        if info.get('transNames'):
+            alt_title = '/'.join(info.get('transNames'))
 
         return {
             'id': song_id,
@@ -294,7 +306,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             mv_id, 'Downloading mv info')['data']
 
         formats = [
-            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'preference': int(brs)}
+            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
             for brs, mv_url in info['brs'].items()
         ]
         self._sort_formats(formats)

From 2b0fa1f7dd8d158b69eec4d17b254b99e976bc5c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 15:09:12 +0800
Subject: [PATCH 1116/1515] [kuwo] Merge KuwoSingerMusicIE into KuwoSingerIE

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 82cd85c44..a348b3077 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -262,7 +262,6 @@ from .kuwo import (
     KuwoAlbumIE,
     KuwoChartIE,
     KuwoSingerIE,
-    KuwoSingerMusicIE,
     KuwoCategoryIE,
     KuwoMvIE,
 )

From 1633491bff3e393d7d095b7303f954dacce4f4da Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 15:19:07 +0800
Subject: [PATCH 1117/1515] [kuwo] Merge KuwoSingerMusicIE into KuwoSingerIE
 (missed kuwo.py)

---
 youtube_dl/extractor/kuwo.py | 53 ++++++++++++------------------------
 1 file changed, 17 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 6a96a1aa4..82d5f3f95 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -180,15 +180,22 @@ class KuwoChartIE(InfoExtractor):
 
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/$'
-    _TEST = {
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
             'id': 'bruno+mars',
             'title': 'Bruno Mars',
         },
         'playlist_count': 10,
-    }
+    }, {
+        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
+        'info_dict': {
+            'id': 'Ali',
+            'title': 'Ali',
+        },
+        'playlist_mincount': 95,
+    }]
 
     def _real_extract(self, url):
         singer_id = self._match_id(url)
@@ -197,54 +204,28 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'姓名：<span>(.+?)</span>', webpage, 'singer name')
+            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
+        )
 
-        entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>',
-                webpage, flags=re.DOTALL)
-        ]
-        return self.playlist_result(entries, singer_id, singer_name)
-
-
-class KuwoSingerMusicIE(InfoExtractor):
-    IE_NAME = 'kuwo:singermusic'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/music(_[0-9]+)?.htm'
-    _TEST = {
-        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
-        'info_dict': {
-            'id': 'Ali',
-            'title': 'Ali的热门歌曲',
-        },
-        'playlist_mincount': 95,
-    }
-
-    def _real_extract(self, url):
-        singer_id = self._match_id(url)
-
-        list_name = None
         entries = []
+        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
                 singer_id, note='Download song list page #%d' % page_num,
                 errnote='Unable to get song list page #%d' % page_num)
 
-            if list_name is None:
-                list_name = self._html_search_regex(
-                    r'<h1>([^<>]+)<span>', webpage, 'list name')
-
             entries.extend([
                 self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
                     r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
-            ])
-            if not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+            ][:10 if first_page_only else None])
+
+            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
                 break
 
-        return self.playlist_result(entries, singer_id, list_name)
+        return self.playlist_result(entries, singer_id, singer_name)
 
 
 class KuwoCategoryIE(InfoExtractor):

From 15830339eff9076b1c4ca2cb6322fc1ec55233a8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:30:23 +0800
Subject: [PATCH 1118/1515] [neteasemusic] PEP8

---
 youtube_dl/extractor/neteasemusic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 419f3ecc9..e73c77f89 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -52,7 +52,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def convert_milliseconds(cls, ms):
-        return int(round(ms/1000.0))
+        return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
         req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
@@ -146,7 +146,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
         self._sort_formats(formats)
 
         lyrics_info = self.query_api(
-            'song/lyric?id=%s&lv=-1&tv=-1' % song_id, 
+            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
             song_id, 'Downloading lyrics data')
         lyrics = self._process_lyrics(lyrics_info)
 
@@ -183,7 +183,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
         album_id = self._match_id(url)
 
         info = self.query_api(
-            'album/%s?id=%s' % (album_id, album_id), 
+            'album/%s?id=%s' % (album_id, album_id),
             album_id, 'Downloading album data')['album']
 
         name = info['name']
@@ -221,7 +221,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         singer_id = self._match_id(url)
 
         info = self.query_api(
-            'artist/%s?id=%s' % (singer_id, singer_id), 
+            'artist/%s?id=%s' % (singer_id, singer_id),
             singer_id, 'Downloading singer data')
 
         name = info['artist']['name']
@@ -264,7 +264,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         list_id = self._match_id(url)
 
         info = self.query_api(
-            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id, 
+            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
             list_id, 'Downloading playlist data')['result']
 
         name = info['name']
@@ -378,7 +378,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
                 self.to_screen(
                     'Downloading just the main audio %s because of --no-playlist'
                     % info['mainSong']['id'])
-            
+
             formats = self.extract_formats(info['mainSong'])
             self._sort_formats(formats)
 
@@ -429,7 +429,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         entries = []
         for offset in itertools.count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
-                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d' 
+                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),
                 dj_id, 'Downloading dj programs - %d' % offset)
 

From 397a8ea96e228f918f127313f837f7de7dd78aed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:43:38 +0800
Subject: [PATCH 1119/1515] [neteasemusic] Encoding fixes for Python 2.6 and
 3.x

---
 youtube_dl/extractor/neteasemusic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index e73c77f89..9e99ddbda 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -11,6 +11,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_str,
 )
 
 
@@ -21,14 +22,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
-        string_bytes = bytearray(str(dfsid))
+        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
+        string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
         salt_len = len(salt_bytes)
         for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
-        m.update(string_bytes)
-        result = b64encode(m.digest())
+        m.update(bytes(string_bytes))
+        result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
     @classmethod

From a0e060ac1e9871974c02fe7cb3e812a3cbe79e73 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:58:12 +0800
Subject: [PATCH 1120/1515] [compat] Add compat_itertools_count

'step' parameter is added in Python 2.7
---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f9529210d..c3783337a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -9,6 +9,7 @@ import shutil
 import socket
 import subprocess
 import sys
+import itertools
 
 
 try:
@@ -388,6 +389,15 @@ else:
             pass
         return _terminal_size(columns, lines)
 
+try:
+    itertools.count(start=0, step=1)
+    compat_itertools_count = itertools.count
+except TypeError:  # Python 2.6
+    def compat_itertools_count(start=0, step=1):
+        n = start
+        while True:
+            yield n
+            n += step
 
 __all__ = [
     'compat_HTTPError',
@@ -401,6 +411,7 @@ __all__ = [
     'compat_html_entities',
     'compat_http_client',
     'compat_http_server',
+    'compat_itertools_count',
     'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',

From 4eab60cbd27ea8ce31ae9c6bf6b3f6e0e469d3af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:59:12 +0800
Subject: [PATCH 1121/1515] [netease:djradio] Use compat_itertools_count

---
 youtube_dl/extractor/neteasemusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 9e99ddbda..bdfe7e63f 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from hashlib import md5
 from base64 import b64encode
 from datetime import datetime
-import itertools
 import re
 
 from .common import InfoExtractor
@@ -12,6 +11,7 @@ from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
     compat_str,
+    compat_itertools_count,
 )
 
 
@@ -428,7 +428,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         name = None
         desc = None
         entries = []
-        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+        for offset in compat_itertools_count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
                 'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),

From a34af8d0667d8f4ceba3380f808a6d563ca01d77 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 19:13:52 +0800
Subject: [PATCH 1122/1515] [kuwo] PEP8

---
 youtube_dl/extractor/kuwo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 82d5f3f95..9c62191b5 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -249,10 +249,10 @@ class KuwoCategoryIE(InfoExtractor):
 
         category_name = self._html_search_regex(
             r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
-        
+
         category_desc = re.sub(
             r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
-        
+
         jsonm = self._parse_json(self._html_search_regex(
             r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
 

From 9296e92e1cbb5788c18c9a09590ccbc0e3faec68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 20:55:53 +0600
Subject: [PATCH 1123/1515] [twitch] Fix login (Closes #6186)

---
 youtube_dl/extractor/twitch.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b56ee2959..b7a72a7bd 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -22,8 +22,8 @@ class TwitchBaseIE(InfoExtractor):
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
-    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
+    _LOGIN_URL = 'https://secure.twitch.tv/login'
+    _LOGIN_POST_URL = 'https://passport.twitch.tv/authorize'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -59,20 +59,14 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        authenticity_token = self._search_regex(
-            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
-            login_page, 'authenticity token')
+        login_form = dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            login_page))
 
-        login_form = {
-            'utf8': '✓'.encode('utf-8'),
-            'authenticity_token': authenticity_token,
-            'redirect_on_login': '',
-            'embed_form': 'false',
-            'mp_source_action': 'login-button',
-            'follow': '',
+        login_form.update({
             'login': username,
             'password': password,
-        }
+        })
 
         request = compat_urllib_request.Request(
             self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))

From 89a683ae745de77f96ac8e82c8ff16d2675fa398 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:15:09 +0600
Subject: [PATCH 1124/1515] [twitch] Fix error message regex

---
 youtube_dl/extractor/twitch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b7a72a7bd..f912d3825 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -74,11 +74,12 @@ class TwitchBaseIE(InfoExtractor):
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        m = re.search(
-            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
-        if m:
+        error_message = self._search_regex(
+            r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
+            response, 'error message', default=None)
+        if error_message:
             raise ExtractorError(
-                'Unable to login: %s' % m.group('msg').strip(), expected=True)
+                'Unable to login. Twitch said: %s' % error_message, expected=True)
 
     def _prefer_source(self, formats):
         try:

From 17b41a3337b5326fd2c8dec8aa1ab285225e1daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:16:42 +0600
Subject: [PATCH 1125/1515] [twitch] Show reset password request

---
 youtube_dl/extractor/twitch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f912d3825..df809ecfe 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -81,6 +81,9 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login. Twitch said: %s' % error_message, expected=True)
 
+        if '>Reset your password<' in response:
+            self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
+
     def _prefer_source(self, formats):
         try:
             source = next(f for f in formats if f['format_id'] == 'Source')

From a31e3e7dcb9d0471d90ec8562934a144d25d7132 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:23:07 +0800
Subject: [PATCH 1126/1515] [kuwo] Regular expression improvements

1. Prevent .+ and .*
2. Use [^>]+ instead of spaces for HTML tags
3. Remove unnecessary trailing parts
---
 youtube_dl/extractor/kuwo.py         | 47 +++++++++++++++-------------
 youtube_dl/extractor/neteasemusic.py |  2 +-
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 9c62191b5..1095a26e2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -9,6 +9,7 @@ from ..utils import (
     get_element_by_id,
     clean_html,
     ExtractorError,
+    remove_start,
 )
 
 
@@ -73,10 +74,10 @@ class KuwoIE(InfoExtractor):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'<h1 title="(.+?)">', webpage, 'song name')
+            r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
         singer_name = self._html_search_regex(
-            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
-            flags=re.DOTALL, default=None)
+            r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
+            webpage, 'singer name', default=None)
         lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -84,7 +85,7 @@ class KuwoIE(InfoExtractor):
         formats = self._get_formats(song_id)
 
         album_id = self._html_search_regex(
-            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
+            r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
             webpage, 'album id', default=None, fatal=False)
 
         publish_time = None
@@ -131,15 +132,16 @@ class KuwoAlbumIE(InfoExtractor):
             errnote='Unable to get album info')
 
         album_name = self._html_search_regex(
-            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
-            'album name', flags=re.DOTALL)
-        album_intro = clean_html(
-            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
+            r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
+            'album name')
+        album_intro = remove_start(
+            clean_html(get_element_by_id("intro", webpage)),
+            '%s简介：' % album_name)
 
         entries = [
             self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
-                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
+                r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
                 webpage)
         ]
         return self.playlist_result(entries, album_id, album_name, album_intro)
@@ -147,7 +149,7 @@ class KuwoAlbumIE(InfoExtractor):
 
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
@@ -165,15 +167,15 @@ class KuwoChartIE(InfoExtractor):
             errnote='Unable to get chart info')
 
         chart_name = self._html_search_regex(
-            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
+            r'<h1[^>]+class="unDis">([^<]+)</h1>', webpage, 'chart name')
 
         chart_desc = self._html_search_regex(
-            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
+            r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
             self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
-                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
+                r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
         ]
         return self.playlist_result(entries, chart_id, chart_name, chart_desc)
 
@@ -204,11 +206,11 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
+            r'<div class="title clearfix">\s*<h1>([^<]+)<span', webpage, 'singer name'
         )
 
         entries = []
-        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
+        first_page_only = False if re.search(r'/music(?:_[0-9]+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
@@ -218,11 +220,11 @@ class KuwoSingerIE(InfoExtractor):
             entries.extend([
                 self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
-                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                    r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
             ][:10 if first_page_only else None])
 
-            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+            if first_page_only or not re.search(r'<a[^>]+href="[^"]+">下一页</a>', webpage):
                 break
 
         return self.playlist_result(entries, singer_id, singer_name)
@@ -248,13 +250,14 @@ class KuwoCategoryIE(InfoExtractor):
             errnote='Unable to get category info')
 
         category_name = self._html_search_regex(
-            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
+            r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
 
-        category_desc = re.sub(
-            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
+        category_desc = remove_start(
+            get_element_by_id("intro", webpage).strip(),
+            '%s简介：' % category_name)
 
         jsonm = self._parse_json(self._html_search_regex(
-            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
+            r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
 
         entries = [
             self.url_result(
@@ -289,7 +292,7 @@ class KuwoMvIE(KuwoIE):
             errnote='Unable to get mv detail info: %s' % song_id)
 
         mobj = re.search(
-            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
+            r'<h1[^>]+title="(?P<song>[^"]+)">[^<]+<span[^>]+title="(?P<singer>[^"]+)"',
             webpage)
         if mobj:
             song_name = mobj.group('song')
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index bdfe7e63f..ee52efaee 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -229,7 +229,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         if info['artist']['trans']:
             name = '%s - %s' % (name, info['artist']['trans'])
         if info['artist']['alias']:
-            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
 
         entries = [
             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],

From c0bf5e1c4dc0a236402c946b0b1fd358f9e39a74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:40:21 +0600
Subject: [PATCH 1127/1515] [twitch] Fix non-ASCII logins/passwords on python 2

---
 youtube_dl/extractor/twitch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index df809ecfe..49535cd80 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -64,8 +64,8 @@ class TwitchBaseIE(InfoExtractor):
             login_page))
 
         login_form.update({
-            'login': username,
-            'password': password,
+            'login': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
         })
 
         request = compat_urllib_request.Request(

From a9684c0dbf3879478fd223ce7594d58be7dffa4f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:46:44 +0800
Subject: [PATCH 1128/1515] [kuwo] Add KuwoBaseIE

---
 youtube_dl/extractor/kuwo.py | 63 +++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 1095a26e2..928f7f62d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -13,32 +13,7 @@ from ..utils import (
 )
 
 
-class KuwoIE(InfoExtractor):
-    IE_NAME = 'kuwo:song'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
-    _TESTS = [{
-        'url': 'http://www.kuwo.cn/yinyue/635632/',
-        'info_dict': {
-            'id': '635632',
-            'ext': 'ape',
-            'title': '爱我别走',
-            'creator': '张震岳',
-            'upload_date': '20080122',
-            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
-        },
-    }, {
-        'url': 'http://www.kuwo.cn/yinyue/6446136/',
-        'info_dict': {
-            'id': '6446136',
-            'ext': 'mp3',
-            'title': '心',
-            'creator': 'IU',
-            'upload_date': '20150518',
-        },
-        'params': {
-            'format': 'mp3-320'
-        },
-    }]
+class KuwoBaseIE(InfoExtractor):
     _FORMATS = [
         {'format': 'ape', 'ext': 'ape', 'preference': 100},
         {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
@@ -67,6 +42,34 @@ class KuwoIE(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
+
+class KuwoIE(KuwoBaseIE):
+    IE_NAME = 'kuwo:song'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/yinyue/635632/',
+        'info_dict': {
+            'id': '635632',
+            'ext': 'ape',
+            'title': '爱我别走',
+            'creator': '张震岳',
+            'upload_date': '20080122',
+            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
+        },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/6446136/',
+        'info_dict': {
+            'id': '6446136',
+            'ext': 'mp3',
+            'title': '心',
+            'creator': 'IU',
+            'upload_date': '20150518',
+        },
+        'params': {
+            'format': 'mp3-320'
+        },
+    }]
+
     def _real_extract(self, url):
         song_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -268,10 +271,10 @@ class KuwoCategoryIE(InfoExtractor):
         return self.playlist_result(entries, category_id, category_name, category_desc)
 
 
-class KuwoMvIE(KuwoIE):
+class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
     _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
             'id': '6480076',
@@ -279,8 +282,8 @@ class KuwoMvIE(KuwoIE):
             'title': '我们家MV',
             'creator': '2PM',
         },
-    }]
-    _FORMATS = KuwoIE._FORMATS + [
+    }
+    _FORMATS = KuwoBaseIE._FORMATS + [
         {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
         {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
     ]

From cf2c5fda4f3e753cc64098e6a751cf1a220efae7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:48:48 +0800
Subject: [PATCH 1129/1515] [kuwo] Use single quotes

---
 youtube_dl/extractor/kuwo.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 928f7f62d..2b5321cc2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -27,9 +27,9 @@ class KuwoBaseIE(InfoExtractor):
         formats = []
         for file_format in self._FORMATS:
             song_url = self._download_webpage(
-                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
+                'http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url' %
                 (file_format['ext'], file_format.get('br', ''), song_id),
-                song_id, note="Download %s url info" % file_format["format"],
+                song_id, note='Download %s url info' % file_format['format'],
             )
             if song_url.startswith('http://') or song_url.startswith('https://'):
                 formats.append({
@@ -81,7 +81,7 @@ class KuwoIE(KuwoBaseIE):
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', default=None)
-        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
+        lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
 
@@ -94,7 +94,7 @@ class KuwoIE(KuwoBaseIE):
         publish_time = None
         if album_id is not None:
             album_info_page = self._download_webpage(
-                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
+                'http://www.kuwo.cn/album/%s/' % album_id, song_id,
                 note='Download album detail info',
                 errnote='Unable to get album detail info')
 
@@ -138,11 +138,11 @@ class KuwoAlbumIE(InfoExtractor):
             r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
             'album name')
         album_intro = remove_start(
-            clean_html(get_element_by_id("intro", webpage)),
+            clean_html(get_element_by_id('intro', webpage)),
             '%s简介：' % album_name)
 
         entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
                 r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
                 webpage)
@@ -176,7 +176,7 @@ class KuwoChartIE(InfoExtractor):
             r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
                 r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
         ]
@@ -221,7 +221,7 @@ class KuwoSingerIE(InfoExtractor):
                 errnote='Unable to get song list page #%d' % page_num)
 
             entries.extend([
-                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+                self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
                     r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
@@ -256,7 +256,7 @@ class KuwoCategoryIE(InfoExtractor):
             r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
 
         category_desc = remove_start(
-            get_element_by_id("intro", webpage).strip(),
+            get_element_by_id('intro', webpage).strip(),
             '%s简介：' % category_name)
 
         jsonm = self._parse_json(self._html_search_regex(
@@ -264,7 +264,7 @@ class KuwoCategoryIE(InfoExtractor):
 
         entries = [
             self.url_result(
-                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
+                'http://www.kuwo.cn/yinyue/%s/' % song['musicrid'],
                 'Kuwo', song['musicrid'])
             for song in jsonm['musiclist']
         ]
@@ -301,7 +301,7 @@ class KuwoMvIE(KuwoBaseIE):
             song_name = mobj.group('song')
             singer_name = mobj.group('singer')
         else:
-            raise ExtractorError("Unable to find song or singer names")
+            raise ExtractorError('Unable to find song or singer names')
 
         formats = self._get_formats(song_id)
 

From 27713812a093ddfb329cd9cead878f3d0bf629cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:49:09 +0600
Subject: [PATCH 1130/1515] [extractor/common] Add method for extracting form
 hidden input fields as dict

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d859aea52..82f5de2d8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -705,6 +705,12 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
+    @staticmethod
+    def _form_hidden_inputs(html):
+        return dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            html))
+
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')

From a0aab26a414e8424e852a361c1822739f6bc936c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:49:39 +0600
Subject: [PATCH 1131/1515] [twitch] Use `_form_hidden_inputs` when logging in

---
 youtube_dl/extractor/twitch.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 49535cd80..af2b798fb 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -59,9 +59,7 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        login_form = dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            login_page))
+        login_form = self._form_hidden_inputs(login_page)
 
         login_form.update({
             'login': username.encode('utf-8'),

From a5839317aa5b41e7bc99ecae7405459bbcbff8a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:52:03 +0600
Subject: [PATCH 1132/1515] [vk] Use `_form_hidden_inputs` when logging in

---
 youtube_dl/extractor/vk.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 23d153031..c0292095b 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -154,9 +154,7 @@ class VKIE(InfoExtractor):
         login_page = self._download_webpage(
             'https://vk.com', None, 'Downloading login page')
 
-        login_form = dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            login_page))
+        login_form = self._form_hidden_inputs(login_page)
 
         login_form.update({
             'email': username.encode('cp1251'),

From 2688176c776a2d80c6a4ba85f6e2d07dfd8dba24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:53:38 +0600
Subject: [PATCH 1133/1515] [vodlocker] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/vodlocker.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 1c0966a79..431f4e2e3 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -28,12 +28,7 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!

From d3b8908886a78dd63441754f1ac20b70cb29de56 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:53:48 +0800
Subject: [PATCH 1134/1515] [kuwo] Simpler calls to url_result()

---
 youtube_dl/extractor/kuwo.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 2b5321cc2..69afacac9 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -142,9 +142,8 @@ class KuwoAlbumIE(InfoExtractor):
             '%s简介：' % album_name)
 
         entries = [
-            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
+            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                r'<p[^>]+class="listen"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+/)"',
                 webpage)
         ]
         return self.playlist_result(entries, album_id, album_name, album_intro)
@@ -176,9 +175,8 @@ class KuwoChartIE(InfoExtractor):
             r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
-            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
+            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/"', webpage)
         ]
         return self.playlist_result(entries, chart_id, chart_name, chart_desc)
 
@@ -221,9 +219,8 @@ class KuwoSingerIE(InfoExtractor):
                 errnote='Unable to get song list page #%d' % page_num)
 
             entries.extend([
-                self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-                for song_id in re.findall(
-                    r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                    r'<p[^>]+class="m_name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/',
                     webpage)
             ][:10 if first_page_only else None])
 
@@ -263,9 +260,7 @@ class KuwoCategoryIE(InfoExtractor):
             r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
 
         entries = [
-            self.url_result(
-                'http://www.kuwo.cn/yinyue/%s/' % song['musicrid'],
-                'Kuwo', song['musicrid'])
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song['musicrid'], 'Kuwo')
             for song in jsonm['musiclist']
         ]
         return self.playlist_result(entries, category_id, category_name, category_desc)

From de195c23a6837d1526198320561feba74886b9df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:54:39 +0600
Subject: [PATCH 1135/1515] [vimeo] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/vimeo.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cae90205d..d63c03183 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -452,11 +452,7 @@ class VimeoChannelIE(InfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', login_form))
+        fields = self._form_hidden_inputs(login_form)
         token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password

From 3b16d803c9633b7f3653d3d6d8022ed8642a09d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:55:22 +0600
Subject: [PATCH 1136/1515] [shared] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/shared.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 9f3e944e7..7fb68bc2d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -35,8 +35,7 @@ class SharedIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
-        download_form = dict(re.findall(
-            r'<input type="hidden" name="([^"]+)" value="([^"]*)"', webpage))
+        download_form = self._form_hidden_inputs(webpage)
         request = compat_urllib_request.Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')

From b8070dbbd72fa24fb4c0b7973bf547825a0cfed3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:56:14 +0600
Subject: [PATCH 1137/1515] [promptfile] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/promptfile.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index f536e6e6c..81a63c7fc 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -35,10 +35,7 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        fields = dict(re.findall(r'''(?x)type="hidden"\s+
-            name="(.+?)"\s+
-            value="(.*?)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
         req = compat_urllib_request.Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')

From 46f0f500163f64ed8276466667d6733e036bfa2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:56:41 +0600
Subject: [PATCH 1138/1515] [primesharetv] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/primesharetv.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 01cc3d9ea..94c9fb2cb 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -31,12 +31,7 @@ class PrimeShareTVIE(InfoExtractor):
         if '>File not exist<' in webpage:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         headers = {
             'Referer': url,

From 9f01c1a803d06034d443882f57528eba342a9c94 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:56:51 +0800
Subject: [PATCH 1139/1515] [kuwo] Use \d instead of [0-9]

---
 youtube_dl/extractor/kuwo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 69afacac9..18bf66404 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -45,7 +45,7 @@ class KuwoBaseIE(InfoExtractor):
 
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -116,7 +116,7 @@ class KuwoIE(KuwoBaseIE):
 
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
-    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
@@ -157,7 +157,7 @@ class KuwoChartIE(InfoExtractor):
         'info_dict': {
             'id': '香港中文龙虎榜',
             'title': '香港中文龙虎榜',
-            'description': 're:[0-9]{4}第[0-9]{2}期',
+            'description': 're:\d{4}第\d{2}期',
         },
         'playlist_mincount': 10,
     }
@@ -211,7 +211,7 @@ class KuwoSingerIE(InfoExtractor):
         )
 
         entries = []
-        first_page_only = False if re.search(r'/music(?:_[0-9]+)?\.htm', url) else True
+        first_page_only = False if re.search(r'/music(?:_\d+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
@@ -232,7 +232,7 @@ class KuwoSingerIE(InfoExtractor):
 
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
         'info_dict': {
@@ -268,7 +268,7 @@ class KuwoCategoryIE(InfoExtractor):
 
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
-    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {

From 01b89d5682adaecfd319897683b2043cfc7344c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:57:29 +0600
Subject: [PATCH 1140/1515] [played] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/played.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 45716c75d..9fe1524f2 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -38,9 +38,7 @@ class PlayedIE(InfoExtractor):
         if m_error:
             raise ExtractorError(m_error.group('msg'), expected=True)
 
-        fields = re.findall(
-            r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
-        data = dict(fields)
+        data = self._form_hidden_inputs(orig_webpage)
 
         self._sleep(2, video_id)
 

From 8fa7e5817a37756e4135fcb9daba47fc543be36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:58:24 +0600
Subject: [PATCH 1141/1515] [hostingbulk] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/hostingbulk.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 704d0285d..63f579592 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -58,11 +58,7 @@ class HostingBulkIE(InfoExtractor):
             r'<img src="([^"]+)".+?class="pic"',
             webpage, 'thumbnail', fatal=False)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         request = compat_urllib_request.Request(url, urlencode_postdata(fields))
         request.add_header('Content-type', 'application/x-www-form-urlencoded')

From 1c0163a5cc1d6b8fa5c7b4ee39b28e6e47c8981c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:58:49 +0600
Subject: [PATCH 1142/1515] [gorillavid] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/gorillavid.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 6147596e4..aabf07a20 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -78,12 +78,7 @@ class GorillaVidIE(InfoExtractor):
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             countdown = int_or_none(self._search_regex(

From 094790d2c963ef7ee9f5861e0458174ddd15ed87 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:03:49 +0800
Subject: [PATCH 1143/1515] [kuwo:song] Give warnings for unavailable optional
 fields

---
 youtube_dl/extractor/kuwo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 18bf66404..a021f3cdf 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -80,7 +80,7 @@ class KuwoIE(KuwoBaseIE):
             r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
-            webpage, 'singer name', default=None)
+            webpage, 'singer name', fatal=False)
         lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -89,7 +89,7 @@ class KuwoIE(KuwoBaseIE):
 
         album_id = self._html_search_regex(
             r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
-            webpage, 'album id', default=None, fatal=False)
+            webpage, 'album id', fatal=False)
 
         publish_time = None
         if album_id is not None:
@@ -100,7 +100,7 @@ class KuwoIE(KuwoBaseIE):
 
             publish_time = self._html_search_regex(
                 r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
-                'publish time', default=None)
+                'publish time', fatal=False)
             if publish_time:
                 publish_time = publish_time.replace('-', '')
 

From 446e7645001d276e6036b9a81e7996edc93b360b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:49:59 +0800
Subject: [PATCH 1144/1515] [baidu] Add localized name

---
 youtube_dl/extractor/baidu.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 906895c1e..e37ee4440 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
+    IE_DESC = '百度视频'
     _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',

From e1ba152352e95fb4cb6f3a5025166627d9df9787 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:50:31 +0800
Subject: [PATCH 1145/1515] [ctsnews] Add localized name

---
 youtube_dl/extractor/ctsnews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 0226f8036..45049bf37 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -6,6 +6,7 @@ from ..utils import parse_iso8601, ExtractorError
 
 
 class CtsNewsIE(InfoExtractor):
+    IE_DESC = '華視新聞'
     # https connection failed (Connection reset)
     _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{

From 513cbdda935b9f319cb61bffcdb0107249eb6b78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:50:45 +0800
Subject: [PATCH 1146/1515] [douyutv] Add localized name

---
 youtube_dl/extractor/douyutv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 479430c51..373b3b4b4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -9,6 +9,7 @@ from ..compat import (compat_str, compat_basestring)
 
 
 class DouyuTVIE(InfoExtractor):
+    IE_DESC = '斗鱼'
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',

From 44c514eb9c03ddf495feeaf94495bee39cffd842 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:03 +0800
Subject: [PATCH 1147/1515] [iqiyi] Add localized name

---
 youtube_dl/extractor/iqiyi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 9106dd074..0f6707d7c 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -20,6 +20,7 @@ from ..utils import (
 
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
+    IE_DESC = '爱奇艺'
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
 

From 0f08d7f851d8e41e09326d5e55dd93038d3ec752 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:14 +0800
Subject: [PATCH 1148/1515] [kuwo] Add localized name

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a021f3cdf..43f616a5d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -45,6 +45,7 @@ class KuwoBaseIE(InfoExtractor):
 
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
+    IE_DESC = '酷我音乐'
     _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',

From 963d0ce7e342841378945a8cb58d3c30bca02c60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:26 +0800
Subject: [PATCH 1149/1515] [letv] Add localized name

---
 youtube_dl/extractor/letv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index da896caf1..ba2ae8085 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 
 class LetvIE(InfoExtractor):
+    IE_DESC = '乐视网'
     _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
 
     _TESTS = [{

From 6ce89aecc330fe4b812526da4e119e7587548e27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:35 +0800
Subject: [PATCH 1150/1515] [neteasemusic] Add localized name

---
 youtube_dl/extractor/neteasemusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index ee52efaee..8758a6f0b 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -63,6 +63,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
 class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:song'
+    IE_DESC = '网易云音乐'
     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',

From 9d16788ad928800f4b6c5682dd08d0814a98754b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:55 +0800
Subject: [PATCH 1151/1515] [nextmedia] Add localized name

---
 youtube_dl/extractor/nextmedia.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index d1b7cff4c..c10784f6b 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -6,6 +6,7 @@ from ..utils import parse_iso8601
 
 
 class NextMediaIE(InfoExtractor):
+    IE_DESC = '蘋果日報'
     _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
@@ -66,6 +67,7 @@ class NextMediaIE(InfoExtractor):
 
 
 class NextMediaActionNewsIE(NextMediaIE):
+    IE_DESC = '蘋果日報 - 動新聞'
     _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
     _TESTS = [{
         'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
@@ -90,6 +92,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 
 class AppleDailyIE(NextMediaIE):
+    IE_DESC = '臺灣蘋果日報'
     _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',

From a7ada46bd90666b9c351bb54bd63b7225307afed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:07 +0800
Subject: [PATCH 1152/1515] [qqmusic] Add localized name

---
 youtube_dl/extractor/qqmusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 476432330..4c05fded4 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -16,6 +16,7 @@ from ..compat import compat_urllib_request
 
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
+    IE_DESC = 'QQ音乐'
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',

From 9b15be97aa0e094cccaf13e181b51b4399347d4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:19 +0800
Subject: [PATCH 1153/1515] [udn] Add localized name

---
 youtube_dl/extractor/udn.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index c08428acf..2151f8338 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -11,6 +11,7 @@ from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
+    IE_DESC = '聯合影音'
     _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',

From 4fa5f402321c71d6b2d8a281b60b8a23041c0e8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:56 +0800
Subject: [PATCH 1154/1515] [xuite] Add localized name

---
 youtube_dl/extractor/xuite.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 81d885fdc..5aac8adb3 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class XuiteIE(InfoExtractor):
+    IE_DESC = '隨意窩Xuite影音'
     _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
     _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{

From e014ff015d5240bac9371e5ffe5342ce025a574f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:53:08 +0800
Subject: [PATCH 1155/1515] [yam] Add localized name

---
 youtube_dl/extractor/yam.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 9d851bae3..001ee17b6 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class YamIE(InfoExtractor):
+    IE_DESC = '蕃薯藤yam天空部落'
     _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
 
     _TESTS = [{

From b931fbe5aba8e68a822f88d91d4ec424bc6509d3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:57:59 +0800
Subject: [PATCH 1156/1515] [yinyuetai] Add localized name

---
 youtube_dl/extractor/yinyuetai.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index fa6b40816..834d860af 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -7,6 +7,7 @@ from ..utils import ExtractorError
 
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
+    IE_DESC = '音悦Tai'
     _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',

From 246995dbc8ac0eb61355966eba7c7559a9ab7b24 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:58:13 +0800
Subject: [PATCH 1157/1515] [youku] Add localized name

---
 youtube_dl/extractor/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ced3a10cd..78caeb8b3 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -15,6 +15,7 @@ from ..compat import (
 
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
+    IE_DESC = '优酷'
     _VALID_URL = r'''(?x)
         (?:
             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|

From edd66be5be584dfb04d740cdd663fbd6203e3730 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:04 +0800
Subject: [PATCH 1158/1515] [kuwo] Add more localized names

---
 youtube_dl/extractor/kuwo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 43f616a5d..1077846f2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -117,6 +117,7 @@ class KuwoIE(KuwoBaseIE):
 
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
+    IE_DESC = '酷我音乐 - 专辑'
     _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
@@ -152,6 +153,7 @@ class KuwoAlbumIE(InfoExtractor):
 
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
+    IE_DESC = '酷我音乐 - 排行榜'
     _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
@@ -184,6 +186,7 @@ class KuwoChartIE(InfoExtractor):
 
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
+    IE_DESC = '酷我音乐 - 歌手'
     _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
@@ -233,6 +236,7 @@ class KuwoSingerIE(InfoExtractor):
 
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
+    IE_DESC = '酷我音乐 - 分类'
     _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
@@ -269,6 +273,7 @@ class KuwoCategoryIE(InfoExtractor):
 
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
+    IE_DESC = '酷我音乐 - MV'
     _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',

From ed848087d53d09797eec932e84c8c36c010d04a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:18 +0800
Subject: [PATCH 1159/1515] [neteasemusic] Add more localized names

---
 youtube_dl/extractor/neteasemusic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 8758a6f0b..a8e0a64ed 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -171,6 +171,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:album'
+    IE_DESC = '网易云音乐 - 专辑'
     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/album?id=220780',
@@ -200,6 +201,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:singer'
+    IE_DESC = '网易云音乐 - 歌手'
     _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'note': 'Singer has aliases.',
@@ -242,6 +244,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicListIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:playlist'
+    IE_DESC = '网易云音乐 - 歌单'
     _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/playlist?id=79177352',
@@ -287,6 +290,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:mv'
+    IE_DESC = '网易云音乐 - MV'
     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/mv?id=415350',
@@ -327,6 +331,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:program'
+    IE_DESC = '网易云音乐 - 电台节目'
     _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/program?id=10109055',
@@ -411,6 +416,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:djradio'
+    IE_DESC = '网易云音乐 - 电台'
     _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/djradio?id=42',

From 181c4ccaaa186685d4a5fce6f338486182079b9e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:36 +0800
Subject: [PATCH 1160/1515] [qqmusic] Add more localized names

---
 youtube_dl/extractor/qqmusic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 4c05fded4..1654a641f 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -143,6 +143,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
+    IE_DESC = 'QQ音乐 - 歌手'
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -187,6 +188,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
+    IE_DESC = 'QQ音乐 - 专辑'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TESTS = [{
@@ -229,6 +231,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
+    IE_DESC = 'QQ音乐 - 排行榜'
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
@@ -282,6 +285,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 
 class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:playlist'
+    IE_DESC = 'QQ音乐 - 歌单'
     _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 86f2541695ba0280982e47e52c5cf26946d5d7c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:22:33 +0200
Subject: [PATCH 1161/1515] Remove unused 're' imports

---
 youtube_dl/extractor/primesharetv.py | 2 --
 youtube_dl/extractor/shared.py       | 1 -
 youtube_dl/extractor/vodlocker.py    | 2 --
 3 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 94c9fb2cb..9aa0c862a 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 7fb68bc2d..6e2b94e7d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 431f4e2e3..4804692bf 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,

From 0a31a350981931ab9403d58258f5058ed98142a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:46:25 +0200
Subject: [PATCH 1162/1515] [YoutubeDL] format spec: add additional checks for
 invalid syntax

---
 test/test_YoutubeDL.py  | 10 ++++++++++
 youtube_dl/YoutubeDL.py |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bf2baae07..20f45f439 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -302,6 +302,16 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1['format_id'])
 
+    def test_invalid_format_specs(self):
+        def assert_syntax_error(format_spec):
+            ydl = YDL({'format': format_spec})
+            info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
+            self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
+
+        assert_syntax_error('bestvideo,,best')
+        assert_syntax_error('+bestaudio')
+        assert_syntax_error('bestvideo+')
+
     def test_format_filtering(self):
         formats = [
             {'format_id': 'A', 'filesize': 500, 'width': 1000},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6478d05dc..da7c51008 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -953,6 +953,8 @@ class YoutubeDL(object):
                         tokens.restore_last_token()
                         break
                     elif string == ',':
+                        if not current_selector:
+                            raise syntax_error('"," must follow a format selector', start)
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
@@ -972,6 +974,8 @@ class YoutubeDL(object):
                     elif string == '+':
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
+                        if not video_selector or not audio_selector:
+                            raise syntax_error('"+" must be between two format selectors', start)
                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                     else:
                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)

From dc48695ab9c84cda56ee723aee11e5da44c2258a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:59:45 +0200
Subject: [PATCH 1163/1515] Document how to group format selectors

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e3452c9e1..4e6cb3fc7 100644
--- a/README.md
+++ b/README.md
@@ -268,7 +268,7 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
 Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 

From fac54cb426d85572c6928ce0d9e5bf1efb459548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 04:43:29 +0600
Subject: [PATCH 1164/1515] [webofstories:playlist] Improve and add test

---
 youtube_dl/extractor/webofstories.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index d70e30c00..2037d9b3d 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -104,7 +104,14 @@ class WebOfStoriesIE(InfoExtractor):
 
 class WebOfStoriesPlaylistIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
-    _TESTS = []
+    _TEST = {
+        'url': 'http://www.webofstories.com/playAll/donald.knuth',
+        'info_dict': {
+            'id': 'donald.knuth',
+            'title': 'Donald Knuth (Scientist)',
+        },
+        'playlist_mincount': 97,
+    }
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -116,10 +123,19 @@ class WebOfStoriesPlaylistIE(InfoExtractor):
             for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
         ]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Web\sof\sStories</title>', webpage, 'title')
+        title = self._search_regex(
+            r'<div id="speakerName">\s*<span>([^<]+)</span>',
+            webpage, 'speaker', default=None)
+        if title:
+            field = self._search_regex(
+                r'<span id="primaryField">([^<]+)</span>',
+                webpage, 'field', default=None)
+            if field:
+                title += ' (%s)' % field
 
-        description = self._html_search_meta(
-            'description', webpage, 'description')
+        if not title:
+            title = self._search_regex(
+                r'<title>Play\s+all\s+stories\s*-\s*([^<]+)\s*-\s*Web\s+of\s+Stories</title>',
+                webpage, 'title')
 
-        return self.playlist_result(entries, playlist_id, title, description)
+        return self.playlist_result(entries, playlist_id, title)

From 79913fde35ead6247d7b764f3a95059fc4f17dbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 21:23:49 +0600
Subject: [PATCH 1165/1515] [vk] Add list id to info_url

---
 youtube_dl/extractor/vk.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c0292095b..333c2a634 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -29,7 +29,7 @@ class VKIE(InfoExtractor):
                                 (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
                                 (?:www\.)?biqle\.ru/watch/
                             )
-                            (?P<videoid>[^s].*?)(?:\?|%2F|$)
+                            (?P<videoid>[^s].*?)(?:\?(?:.*\blist=(?P<list_id>[\da-f]+))?|%2F|$)
                         )
                     '''
     _NETRC_MACHINE = 'vk'
@@ -119,6 +119,20 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # video (removed?) only available with list id
+            'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4',
+            'md5': '091287af5402239a1051c37ec7b92913',
+            'info_dict': {
+                'id': '171201961',
+                'ext': 'mp4',
+                'title': 'ТюменцевВВ_09.07.2015',
+                'uploader': 'Anton Ivanov',
+                'duration': 109,
+                'upload_date': '20150709',
+                'view_count': int,
+            },
+        },
         {
             # youtube embed
             'url': 'https://vk.com/video276849682_170681728',
@@ -182,6 +196,12 @@ class VKIE(InfoExtractor):
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
         info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+
+        # Some videos (removed?) can only be downloaded with list id specified
+        list_id = mobj.group('list_id')
+        if list_id:
+            info_url += '&list=%s' % list_id
+
         info_page = self._download_webpage(info_url, video_id)
 
         if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):

From d919fa3344463f94a152a10ba65981cd290d9ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 21:26:03 +0600
Subject: [PATCH 1166/1515] [vk] Handle access denied error

---
 youtube_dl/extractor/vk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 333c2a634..8ac3aeac0 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -222,6 +222,9 @@ class VKIE(InfoExtractor):
 
             r'<!>Видео временно недоступно':
             'Video %s is temporarily unavailable.',
+
+            r'<!>Access denied':
+            'Access denied to video %s.',
         }
 
         for error_re, error_msg in ERRORS.items():

From f72b0a603270dff0fdd72dd5218126790232199a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:15:16 +0600
Subject: [PATCH 1167/1515] Revert xvid to avi and make docs to be similar to
 existing external downloader option

---
 README.md                          | 4 ++--
 youtube_dl/YoutubeDL.py            | 3 ++-
 youtube_dl/__init__.py             | 2 +-
 youtube_dl/options.py              | 6 +++---
 youtube_dl/postprocessor/ffmpeg.py | 8 +++-----
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index d8926d2b7..9779c2058 100644
--- a/README.md
+++ b/README.md
@@ -214,8 +214,8 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --postprocessor-args             Extra parameters for video post-processor.
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)
+    --postprocessor-args ARGS        Give these arguments to the postprocessor
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8580f99a7..00af78e06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -262,7 +262,8 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    postprocessor_args: Extra parameters for external apps, like avconv.
+    postprocessor_args: A list of additional command-line arguments for the
+                        postprocessor.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index fb31d1569..2d416943f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -169,7 +169,7 @@ def _real_main(argv=None):
         if not opts.audioquality.isdigit():
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
-        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
             parser.error('invalid video recode format specified')
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c15dadb21..85365d769 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -691,11 +691,11 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
-        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)')
     postproc.add_option(
         '--postprocessor-args',
-        dest='postprocessor_args', default=None, metavar='ARGS',
-        help='Extra parameters for video post-processor.')
+        dest='postprocessor_args', metavar='ARGS',
+        help='Give these arguments to the postprocessor')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d4ba3572b..62d13a567 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -295,12 +295,10 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
-        ext = self._preferedformat
+        outpath = prefix + sep + self._preferedformat
         options = self._extra_cmd_args
-        if self._preferedformat == 'xvid':
-            ext = 'avi'
+        if self._preferedformat == 'avi':
             options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
-        outpath = prefix + sep + ext
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
@@ -308,7 +306,7 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
         self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath
         information['format'] = self._preferedformat
-        information['ext'] = ext
+        information['ext'] = self._preferedformat
         return [path], information
 
 

From e35b23f54df83886dd14428b618aa9d0221968ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:41:33 +0600
Subject: [PATCH 1168/1515] [postprocessor/common] Improve postprocessor args
 fetching and clarify doc

---
 youtube_dl/postprocessor/common.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index bee64c457..4191d040b 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -22,14 +22,15 @@ class PostProcessor(object):
     of the chain is reached.
 
     PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects. And it can receive parameters from CLI trough
-    --postprocessor-args.
+    to InfoExtractor objects.
+
+    Optionally PostProcessor can use a list of additional command-line arguments
+    with self._configuration_args.
     """
 
     _downloader = None
 
     def __init__(self, downloader=None):
-        self._extra_cmd_args = downloader.params.get('postprocessor_args')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
@@ -59,6 +60,13 @@ class PostProcessor(object):
         except Exception:
             self._downloader.report_warning(errnote)
 
+    def _configuration_args(self, default=[]):
+        pp_args = self._downloader.params.get('postprocessor_args')
+        if pp_args is None:
+            return default
+        assert isinstance(pp_args, list)
+        return pp_args
+
 
 class AudioConversionError(PostProcessingError):
     pass

From 15006fedb9511599c0e948a982fa282cde74f2ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:42:03 +0600
Subject: [PATCH 1169/1515] [postprocessor/ffmpeg] Spread postprocessor args
 usage on all ffmpeg extractors

---
 youtube_dl/postprocessor/ffmpeg.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 62d13a567..1ecce22e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -131,6 +131,8 @@ class FFmpegPostProcessor(PostProcessor):
         oldest_mtime = min(
             os.stat(encodeFilename(path)).st_mtime for path in input_paths)
 
+        opts += self._configuration_args()
+
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
@@ -294,14 +296,14 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
     def run(self, information):
         path = information['filepath']
-        prefix, sep, ext = path.rpartition('.')
-        outpath = prefix + sep + self._preferedformat
-        options = self._extra_cmd_args
-        if self._preferedformat == 'avi':
-            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
+        options = []
+        if self._preferedformat == 'avi':
+            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
+        prefix, sep, ext = path.rpartition('.')
+        outpath = prefix + sep + self._preferedformat
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath

From 369e195a44530b43bfd0e95087c9e1a238d91184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:43:02 +0600
Subject: [PATCH 1170/1515] Handle postprocessor_args similarly to
 external_downloader_args

---
 youtube_dl/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2d416943f..55b22c889 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -263,6 +263,9 @@ def _real_main(argv=None):
     external_downloader_args = None
     if opts.external_downloader_args:
         external_downloader_args = shlex.split(opts.external_downloader_args)
+    postprocessor_args = None
+    if opts.postprocessor_args:
+        postprocessor_args = shlex.split(opts.postprocessor_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))
@@ -354,7 +357,6 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
-        'postprocessor_args': shlex.split(opts.postprocessor_args or ''),
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
@@ -368,6 +370,7 @@ def _real_main(argv=None):
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'external_downloader_args': external_downloader_args,
+        'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
     }
 

From 41c0d2f8cb22fe34d957bc9b5f9032a9160685ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 23:00:19 +0600
Subject: [PATCH 1171/1515] Credit @aurium for avi recode and postprocessort
 args (#5942)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d5418dd37..c0201d875 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -130,3 +130,4 @@ Peter Ding
 jackyzy823
 George Brighton
 Remita Amine
+Aurélio A. Heckert

From ac8f97f2b37115bf81a653360819b5d30eebda18 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 11 Jul 2015 18:56:07 +0200
Subject: [PATCH 1172/1515] [rdsca] New extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rdsca.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/rdsca.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..73fa0e4ed 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,6 +460,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
+from .rdsca import RDScaIE
 from .redtube import RedTubeIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
diff --git a/youtube_dl/extractor/rdsca.py b/youtube_dl/extractor/rdsca.py
new file mode 100644
index 000000000..a8df927e4
--- /dev/null
+++ b/youtube_dl/extractor/rdsca.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    url_basename,
+)
+
+
+class RDScaIE(InfoExtractor):
+    IE_NAME = 'RDS.ca'
+    _VALID_URL = r'http://(?:www\.)?rds\.ca/videos/(?P<id>.*)'
+
+    _TESTS = [{
+        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        'info_dict': {
+            "ext": "mp4",
+            "title": "Fowler Jr. prend la direction de Jacksonville",
+            "description": "Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ",
+            "timestamp": 1430397346,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = url_basename(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'<span itemprop="name"[^>]*>([^\n]*)</span>', webpage, 'video title', default=None)
+        video_url = self._search_regex(
+            r'<span itemprop="contentURL" content="([^"]+)"', webpage, 'video URL')
+        upload_date = parse_iso8601(self._search_regex(
+            r'<span itemprop="uploadDate" content="([^"]+)"', webpage, 'upload date', default=None))
+        description = self._search_regex(
+            r'<span itemprop="description"[^>]*>([^\n]*)</span>', webpage, 'description', default=None)
+        thumbnail = self._search_regex(
+            r'<span itemprop="thumbnailUrl" content="([^"]+)"', webpage, 'upload date', default=None)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': upload_date,
+            'formats': [{
+                'url': video_url,
+            }],
+        }

From 53429e6551b80f78273f5245e11613fd39625113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 12:53:15 +0600
Subject: [PATCH 1173/1515] [onionstudios] Fix extraction

---
 youtube_dl/extractor/onionstudios.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 8fa507dec..0f1f448fe 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -49,19 +49,21 @@ class OnionStudiosIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._search_regex(
-            r'share_title\s*=\s*"([^"]+)"', webpage, 'title')
+            r'share_title\s*=\s*(["\'])(?P<title>[^\1]+?)\1',
+            webpage, 'title', group='title')
         description = self._search_regex(
-            r'share_description\s*=\s*"([^"]+)"', webpage,
-            'description', default=None)
+            r'share_description\s*=\s*(["\'])(?P<description>[^\1]+?)\1',
+            webpage, 'description', default=None, group='description')
         thumbnail = self._search_regex(
-            r'poster="([^"]+)"', webpage, 'thumbnail', default=False)
+            r'poster\s*=\s*(["\'])(?P<thumbnail>[^\1]+?)\1',
+            webpage, 'thumbnail', default=False, group='thumbnail')
 
         uploader_id = self._search_regex(
-            r'twitter_handle\s*=\s*"([^"]+)"',
-            webpage, 'uploader id', fatal=False)
+            r'twitter_handle\s*=\s*(["\'])(?P<uploader_id>[^\1]+?)\1',
+            webpage, 'uploader id', fatal=False, group='uploader_id')
         uploader = self._search_regex(
-            r'window\.channelName\s*=\s*"Embedded:([^"]+)"',
-            webpage, 'uploader', default=False)
+            r'window\.channelName\s*=\s*(["\'])Embedded:(?P<uploader>[^\1]+?)\1',
+            webpage, 'uploader', default=False, group='uploader')
 
         return {
             'id': video_id,

From 667170e2c7dee6bcf5e357dd53d461807434c5b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 20:40:00 +0600
Subject: [PATCH 1174/1515] [nrk:tv] Add support for radio URLs (Closes #6200)

---
 youtube_dl/extractor/nrk.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 9e4581cf9..eb49f292e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -116,7 +116,8 @@ class NRKPlaylistIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>https?://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    IE_DESC = 'NRK TV and NRK Radio'
+    _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -188,6 +189,10 @@ class NRKTVIE(InfoExtractor):
                 'duration': 6947.5199999999995,
             },
             'skip': 'Only works from Norway',
+        },
+        {
+            'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
+            'only_matching': True,
         }
     ]
 

From c4f1fde75bf69e05be29283c9211089e6ab269ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 20:42:18 +0600
Subject: [PATCH 1175/1515] [nrk:tv] Add format id prefixes

---
 youtube_dl/extractor/nrk.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index eb49f292e..d066a96db 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -211,7 +211,8 @@ class NRKTVIE(InfoExtractor):
         ]}
 
     def _extract_f4m(self, manifest_url, video_id):
-        return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
+        return self._extract_f4m_formats(
+            manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -273,7 +274,7 @@ class NRKTVIE(InfoExtractor):
 
         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
+            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4', m3u8_id='hls'))
         self._sort_formats(formats)
 
         subtitles_url = self._html_search_regex(

From b6ea9ef21ad3a43bddf24f5769bdf21a372381fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 23:35:56 +0600
Subject: [PATCH 1176/1515] [rds] Improve

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rds.py      | 70 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/rdsca.py    | 50 -----------------------
 3 files changed, 71 insertions(+), 51 deletions(-)
 create mode 100644 youtube_dl/extractor/rds.py
 delete mode 100644 youtube_dl/extractor/rdsca.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 73fa0e4ed..3f4f23521 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,7 +460,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
-from .rdsca import RDScaIE
+from .rds import RDSIE
 from .redtube import RedTubeIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
new file mode 100644
index 000000000..fdd3f36a6
--- /dev/null
+++ b/youtube_dl/extractor/rds.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class RDSIE(InfoExtractor):
+    IE_DESC = 'RDS.ca'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/videos/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+
+    _TEST = {
+        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        'info_dict': {
+            'id': '3.1132799',
+            'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
+            'ext': 'mp4',
+            'title': 'Fowler Jr. prend la direction de Jacksonville',
+            'description': 'Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ',
+            'timestamp': 1430397346,
+            'upload_date': '20150430',
+            'duration': 154.354,
+            'age_limit': 0,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        # TODO: extract f4m from 9c9media.com
+        video_url = self._search_regex(
+            r'<span[^>]+itemprop="contentURL"[^>]+content="([^"]+)"',
+            webpage, 'video url')
+
+        title = self._og_search_title(webpage) or self._html_search_meta(
+            'title', webpage, 'title', fatal=True)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage) or self._search_regex(
+            [r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
+             r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
+            webpage, 'thumbnail', fatal=False)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<span[^>]+itemprop="uploadDate"[^>]+content="([^"]+)"',
+            webpage, 'upload date', fatal=False))
+        duration = parse_duration(self._search_regex(
+            r'<span[^>]+itemprop="duration"[^>]+content="([^"]+)"',
+            webpage, 'duration', fatal=False))
+        age_limit = self._family_friendly_search(webpage)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'age_limit': age_limit,
+        }
diff --git a/youtube_dl/extractor/rdsca.py b/youtube_dl/extractor/rdsca.py
deleted file mode 100644
index a8df927e4..000000000
--- a/youtube_dl/extractor/rdsca.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    url_basename,
-)
-
-
-class RDScaIE(InfoExtractor):
-    IE_NAME = 'RDS.ca'
-    _VALID_URL = r'http://(?:www\.)?rds\.ca/videos/(?P<id>.*)'
-
-    _TESTS = [{
-        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
-        'info_dict': {
-            "ext": "mp4",
-            "title": "Fowler Jr. prend la direction de Jacksonville",
-            "description": "Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ",
-            "timestamp": 1430397346,
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = url_basename(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._search_regex(
-            r'<span itemprop="name"[^>]*>([^\n]*)</span>', webpage, 'video title', default=None)
-        video_url = self._search_regex(
-            r'<span itemprop="contentURL" content="([^"]+)"', webpage, 'video URL')
-        upload_date = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate" content="([^"]+)"', webpage, 'upload date', default=None))
-        description = self._search_regex(
-            r'<span itemprop="description"[^>]*>([^\n]*)</span>', webpage, 'description', default=None)
-        thumbnail = self._search_regex(
-            r'<span itemprop="thumbnailUrl" content="([^"]+)"', webpage, 'upload date', default=None)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': upload_date,
-            'formats': [{
-                'url': video_url,
-            }],
-        }

From 28fb109ed0276e85207966ac925e9885c2ed0e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 23:45:47 +0600
Subject: [PATCH 1177/1515] [rds] Improve _VALID_URL

---
 youtube_dl/extractor/rds.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
index fdd3f36a6..796adfdf9 100644
--- a/youtube_dl/extractor/rds.py
+++ b/youtube_dl/extractor/rds.py
@@ -12,9 +12,9 @@ from ..utils import (
 
 class RDSIE(InfoExtractor):
     IE_DESC = 'RDS.ca'
-    _VALID_URL = r'https?://(?:www\.)?rds\.ca/videos/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
         'info_dict': {
             'id': '3.1132799',
@@ -27,7 +27,10 @@ class RDSIE(InfoExtractor):
             'duration': 154.354,
             'age_limit': 0,
         }
-    }
+    }, {
+        'url': 'http://www.rds.ca/vid%C3%A9os/un-voyage-positif-3.877934',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 75a40b22511b4d4344cd6903e28429fa6c3991f1 Mon Sep 17 00:00:00 2001
From: Seamus Phelan <SeamusPhelan@gmail.com>
Date: Mon, 13 Jul 2015 22:35:45 +1000
Subject: [PATCH 1178/1515] [SBS] fixes due to website changes

---
 youtube_dl/extractor/sbs.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index d4bd1a0d7..ab4d1c884 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -1,12 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    remove_end,
-)
+from ..utils import remove_end
 
 
 class SBSIE(InfoExtractor):
@@ -34,18 +30,22 @@ class SBSIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        # the video is in the following iframe
+        iframe_url = 'http://www.sbs.com.au/ondemand/video/single/' + video_id + '?context=web'
+        webpage = self._download_webpage(iframe_url, video_id)
 
-        player = self._search_regex(
-            r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
-            webpage, 'player')
-        player = re.sub(r"'\s*\+\s*[\da-zA-Z_]+\s*\+\s*'", '', player)
+        player_params = self._search_regex(
+            r'(?s)(playerParams.+?releaseUrls.+?\n)',
+            webpage, 'playerParams')
+        player_params_js = self._search_regex(
+            r'({.*})',
+            player_params, 'player_param_js')
 
-        release_urls = self._parse_json(js_to_json(player), video_id)
+        player_params_json = self._parse_json(player_params_js, video_id)
 
-        theplatform_url = release_urls.get('progressive') or release_urls['standard']
+        theplatform_url = player_params_json.get('releaseUrls')['progressive'] or player_params_json.get('releaseUrls')['standard']
 
-        title = remove_end(self._og_search_title(webpage), ' (The Feed)')
+        title = remove_end(self._og_search_title(webpage, default=video_id, fatal=False), ' (The Feed)')
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 

From 6dfa0602f059397d9a41ef382d9dfa8b432ff0fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jul 2015 22:11:05 +0600
Subject: [PATCH 1179/1515] [nowtv] Fix extraction (Closes #6169)

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 173e46cd8..0b5ff4760 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -133,7 +133,7 @@ class NowTVIE(InfoExtractor):
         station = mobj.group('station')
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files' % display_id,
+            'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
             display_id)
 
         video_id = compat_str(info['id'])

From 7869eb3fc44826533d859d3e7e32df5cd4be7ef4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jul 2015 22:21:35 +0600
Subject: [PATCH 1180/1515] Credit Bernhard Minks for nowtv patch

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c0201d875..531ec5767 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -131,3 +131,4 @@ jackyzy823
 George Brighton
 Remita Amine
 Aurélio A. Heckert
+Bernhard Minks

From cfe5537ee5ca5156f3969f513a1257be83015b1d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Jul 2015 00:21:11 +0800
Subject: [PATCH 1181/1515] [myspass] Fix extraction (closes #6206)

---
 youtube_dl/extractor/myspass.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 5b9b9fbcd..4557a2b13 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -35,7 +35,8 @@ class MySpassIE(InfoExtractor):
 
         # get metadata
         metadata_url = META_DATA_URL_TEMPLATE % video_id
-        metadata = self._download_xml(metadata_url, video_id)
+        metadata = self._download_xml(
+            metadata_url, video_id, transform_source=lambda s: s.strip())
 
         # extract values from metadata
         url_flv_el = metadata.find('url_flv')

From 1d1dd597ed96790b3d041274044742baec13132e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 01:35:44 +0600
Subject: [PATCH 1182/1515] [dramafever] Extract srt (Closes #6207)

---
 youtube_dl/extractor/dramafever.py | 49 +++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index ca41a3abf..38e6597c8 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -23,8 +23,23 @@ class DramaFeverBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
 
+    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
+
+    _consumer_secret = None
+
+    def _get_consumer_secret(self):
+        mainjs = self._download_webpage(
+            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
+            None, 'Downloading main.js', fatal=False)
+        if not mainjs:
+            return self._CONSUMER_SECRET
+        return self._search_regex(
+            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
+            'consumer secret', default=self._CONSUMER_SECRET)
+
     def _real_initialize(self):
         self._login()
+        self._consumer_secret = self._get_consumer_secret()
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -119,6 +134,23 @@ class DramaFeverIE(DramaFeverBaseIE):
                 'url': href,
             }]
 
+        series_id, episode_number = video_id.split('.')
+        episode_info = self._download_json(
+            # We only need a single episode info, so restricting page size to one episode
+            # and dealing with page number as with episode number
+            r'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_number=%s&page_size=1'
+            % (self._consumer_secret, series_id, episode_number),
+            video_id, 'Downloading episode info JSON', fatal=False)
+        if episode_info:
+            value = episode_info.get('value')
+            if value:
+                subfile = value[0].get('subfile') or value[0].get('new_subfile')
+                if subfile and subfile != 'http://www.dramafever.com/st/':
+                    subtitles.setdefault('English', []).append({
+                        'ext': 'srt',
+                        'url': subfile,
+                    })
+
         return {
             'id': video_id,
             'title': title,
@@ -152,27 +184,14 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
         'playlist_count': 20,
     }]
 
-    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
     _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 
-    def _get_consumer_secret(self, video_id):
-        mainjs = self._download_webpage(
-            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
-            video_id, 'Downloading main.js', fatal=False)
-        if not mainjs:
-            return self._CONSUMER_SECRET
-        return self._search_regex(
-            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
-            'consumer secret', default=self._CONSUMER_SECRET)
-
     def _real_extract(self, url):
         series_id = self._match_id(url)
 
-        consumer_secret = self._get_consumer_secret(series_id)
-
         series = self._download_json(
             'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
-            % (consumer_secret, series_id),
+            % (self._consumer_secret, series_id),
             series_id, 'Downloading series JSON')['series'][series_id]
 
         title = clean_html(series['name'])
@@ -182,7 +201,7 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
         for page_num in itertools.count(1):
             episodes = self._download_json(
                 'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
-                % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
+                % (self._consumer_secret, series_id, self._PAGE_SIZE, page_num),
                 series_id, 'Downloading episodes JSON page #%d' % page_num)
             for episode in episodes.get('value', []):
                 episode_url = episode.get('episode_url')

From 50aa2bb6b94805d901f00d4da7a1b4ff1fcd5169 Mon Sep 17 00:00:00 2001
From: cazulu <jvlarapeinado@gmail.com>
Date: Tue, 14 Jul 2015 15:33:55 +0900
Subject: [PATCH 1183/1515] [dailymotion] Extract duration (closes #6221)

---
 youtube_dl/extractor/dailymotion.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 8852f0add..1a41c0db1 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -53,6 +53,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
                 'upload_date': '20150306',
+                'duration': 74,
             }
         },
         # Vevo video
@@ -164,6 +165,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
             'view_count': view_count,
+            'duration': info['duration']
         }
 
     def _get_subtitles(self, video_id, webpage):

From 9750e7d70eed92a6b05637465698cdd30e87a44c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 14 Jul 2015 12:56:32 +0200
Subject: [PATCH 1184/1515] [postprocessor/ffmpeg] Don't use '[youtube] ...' in
 messages

Because it can be used for other extractors.
---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1ecce22e7..1f723908b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -265,7 +265,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
         if (new_path == path or
                 (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
-            self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
+            self._downloader.to_screen('[ffmpeg] Post-process file %s exists, skipping' % new_path)
             return [], information
 
         try:

From f8da79f828637757889f3f35d7adfa9aabbfc721 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:36:30 +0600
Subject: [PATCH 1185/1515] [extractor/common] Improve _form_hidden_inputs and
 rename to _hidden_inputs

---
 youtube_dl/extractor/common.py       | 15 +++++++++++----
 youtube_dl/extractor/gorillavid.py   |  2 +-
 youtube_dl/extractor/hostingbulk.py  |  2 +-
 youtube_dl/extractor/played.py       |  2 +-
 youtube_dl/extractor/primesharetv.py |  2 +-
 youtube_dl/extractor/promptfile.py   |  2 +-
 youtube_dl/extractor/shared.py       |  2 +-
 youtube_dl/extractor/twitch.py       |  2 +-
 youtube_dl/extractor/vimeo.py        |  2 +-
 youtube_dl/extractor/vk.py           |  2 +-
 youtube_dl/extractor/vodlocker.py    |  2 +-
 11 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 82f5de2d8..315fe4a72 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -706,10 +706,17 @@ class InfoExtractor(object):
                                       'twitter card player')
 
     @staticmethod
-    def _form_hidden_inputs(html):
-        return dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            html))
+    def _hidden_inputs(html):
+        return dict([
+            (input.group('name'), input.group('value')) for input in re.finditer(
+                r'''(?x)
+                    <input\s+
+                        type=(?P<q_hidden>["\'])hidden(?P=q_hidden)\s+
+                        name=(?P<q_name>["\'])(?P<name>.+?)(?P=q_name)\s+
+                        (?:id=(?P<q_id>["\']).+?(?P=q_id)\s+)?
+                        value=(?P<q_value>["\'])(?P<value>.*?)(?P=q_value)
+                ''', html)
+        ])
 
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index aabf07a20..f006f0cb1 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -78,7 +78,7 @@ class GorillaVidIE(InfoExtractor):
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             countdown = int_or_none(self._search_regex(
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 63f579592..a3154cfde 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -58,7 +58,7 @@ class HostingBulkIE(InfoExtractor):
             r'<img src="([^"]+)".+?class="pic"',
             webpage, 'thumbnail', fatal=False)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         request = compat_urllib_request.Request(url, urlencode_postdata(fields))
         request.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 9fe1524f2..8a1c296dd 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -38,7 +38,7 @@ class PlayedIE(InfoExtractor):
         if m_error:
             raise ExtractorError(m_error.group('msg'), expected=True)
 
-        data = self._form_hidden_inputs(orig_webpage)
+        data = self._hidden_inputs(orig_webpage)
 
         self._sleep(2, video_id)
 
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 9aa0c862a..304359dc5 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -29,7 +29,7 @@ class PrimeShareTVIE(InfoExtractor):
         if '>File not exist<' in webpage:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         headers = {
             'Referer': url,
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 81a63c7fc..8190ed676 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -35,7 +35,7 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
         req = compat_urllib_request.Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 6e2b94e7d..a07677686 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -34,7 +34,7 @@ class SharedIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
-        download_form = self._form_hidden_inputs(webpage)
+        download_form = self._hidden_inputs(webpage)
         request = compat_urllib_request.Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index af2b798fb..92b6dc1b8 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -59,7 +59,7 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        login_form = self._form_hidden_inputs(login_page)
+        login_form = self._hidden_inputs(login_page)
 
         login_form.update({
             'login': username.encode('utf-8'),
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d63c03183..10d6745af 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -452,7 +452,7 @@ class VimeoChannelIE(InfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
-        fields = self._form_hidden_inputs(login_form)
+        fields = self._hidden_inputs(login_form)
         token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 8ac3aeac0..8f677cae3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -168,7 +168,7 @@ class VKIE(InfoExtractor):
         login_page = self._download_webpage(
             'https://vk.com', None, 'Downloading login page')
 
-        login_form = self._form_hidden_inputs(login_page)
+        login_form = self._hidden_inputs(login_page)
 
         login_form.update({
             'email': username.encode('cp1251'),
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 4804692bf..ccf1928b5 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -26,7 +26,7 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!

From cf61d96df0984e28d8c34328177504a2d1424bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:38:10 +0600
Subject: [PATCH 1186/1515] [extractor/common] Add _form_hidden_inputs

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 315fe4a72..3a396c0b0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -718,6 +718,12 @@ class InfoExtractor(object):
                 ''', html)
         ])
 
+    def _form_hidden_inputs(self, form_id, html):
+        form = self._search_regex(
+            r'(?s)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
+            html, '%s form' % form_id, group='form')
+        return self._hidden_inputs(form)
+
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')

From dcd4d95c8e9acec6dde0706de75bf866a43c9aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:39:41 +0600
Subject: [PATCH 1187/1515] [udemy] Fix authentication (Closes #6224)

---
 youtube_dl/extractor/udemy.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 4667ed83b..192606077 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -15,7 +15,8 @@ from ..utils import (
 class UdemyIE(InfoExtractor):
     IE_NAME = 'udemy'
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
-    _LOGIN_URL = 'https://www.udemy.com/join/login-submit/'
+    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
+    _ORIGIN_URL = 'https://www.udemy.com'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -74,29 +75,34 @@ class UdemyIE(InfoExtractor):
                 expected=True)
 
         login_popup = self._download_webpage(
-            'https://www.udemy.com/join/login-popup?displayType=ajax&showSkipButton=1', None,
-            'Downloading login popup')
+            self._LOGIN_URL, None, 'Downloading login popup')
 
         if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
             return
 
-        csrf = self._html_search_regex(
-            r'<input type="hidden" name="csrf" value="(.+?)"',
-            login_popup, 'csrf token')
+        login_form = self._form_hidden_inputs('login-form', login_popup)
 
-        login_form = {
-            'email': username,
-            'password': password,
-            'csrf': csrf,
+        login_form.update({
             'displayType': 'json',
-            'isSubmitted': '1',
-        }
+            'email': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
+        })
+
         request = compat_urllib_request.Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        response = self._download_json(
+        request.add_header('Referer', self._ORIGIN_URL)
+        request.add_header('Origin', self._ORIGIN_URL)
+
+        response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if 'returnUrl' not in response:
+        if all(logout_pattern not in response
+               for logout_pattern in ['href="https://www.udemy.com/user/logout/', '>Logout<']):
+            error = self._html_search_regex(
+                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
+                response, 'error message', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
             raise ExtractorError('Unable to log in')
 
     def _real_extract(self, url):

From 79057965a87a9179b3c3c2249df5495684c5e4ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:46:26 +0600
Subject: [PATCH 1188/1515] [udemy] Remove superfluous field

---
 youtube_dl/extractor/udemy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 192606077..e2bab52fe 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -83,7 +83,6 @@ class UdemyIE(InfoExtractor):
         login_form = self._form_hidden_inputs('login-form', login_popup)
 
         login_form.update({
-            'displayType': 'json',
             'email': username.encode('utf-8'),
             'password': password.encode('utf-8'),
         })

From 01d115b06b5b752324a9346f128d659a98b58f31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 14 Jul 2015 19:07:40 +0200
Subject: [PATCH 1189/1515] [jeuxvideo] Relax _VALID_URL (fixes #6230)

---
 youtube_dl/extractor/jeuxvideo.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index d0720ff56..1df084d87 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -8,9 +8,9 @@ from .common import InfoExtractor
 
 
 class JeuxVideoIE(InfoExtractor):
-    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)-\d+\.htm'
+    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
         'md5': '046e491afb32a8aaac1f44dd4ddd54ee',
         'info_dict': {
@@ -19,7 +19,10 @@ class JeuxVideoIE(InfoExtractor):
             'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité',
             'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.',
         },
-    }
+    }, {
+        'url': 'http://www.jeuxvideo.com/videos/chroniques/434220/l-histoire-du-jeu-video-la-saturn.htm',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From eae89f92e61d8227a6463c50750310a56c68cf09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 23:54:13 +0600
Subject: [PATCH 1190/1515] [dfb] Update test

---
 youtube_dl/extractor/dfb.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 8049779b0..6732e1751 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -10,13 +10,13 @@ class DFBIE(InfoExtractor):
     _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://tv.dfb.de/video/highlights-des-empfangs-in-berlin/9070/',
+        'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
         # The md5 is different each time
         'info_dict': {
-            'id': '9070',
+            'id': '11633',
             'ext': 'flv',
-            'title': 'Highlights des Empfangs in Berlin',
-            'upload_date': '20140716',
+            'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
+            'upload_date': '20150714',
         },
     }
 

From ddcdc684e20af96a4fc57da49c27b31c75462aae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 23:59:21 +0600
Subject: [PATCH 1191/1515] [dfb] Extract display_id and modernize

---
 youtube_dl/extractor/dfb.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 6732e1751..f1df9b9a0 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -3,17 +3,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import unified_strdate
 
 
 class DFBIE(InfoExtractor):
     IE_NAME = 'tv.dfb.de'
-    _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://tv\.dfb\.de/video/(?P<display_id>[^/]+)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
         # The md5 is different each time
         'info_dict': {
             'id': '11633',
+            'display_id': 'u-19-em-stimmen-zum-spiel-gegen-russland',
             'ext': 'flv',
             'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
             'upload_date': '20150714',
@@ -23,22 +25,25 @@ class DFBIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
         player_info = self._download_xml(
             'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id,
-            video_id)
+            display_id)
         video_info = player_info.find('video')
 
-        f4m_info = self._download_xml(self._proto_relative_url(video_info.find('url').text.strip()), video_id)
+        f4m_info = self._download_xml(
+            self._proto_relative_url(video_info.find('url').text.strip()), display_id)
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': video_info.find('title').text,
             'url': manifest_url,
             'ext': 'flv',
             'thumbnail': self._og_search_thumbnail(webpage),
-            'upload_date': ''.join(video_info.find('time_date').text.split('.')[::-1]),
+            'upload_date': unified_strdate(video_info.find('time_date').text),
         }

From 6c1b0c0ed2678198e3196e2acf32f63806d4ccb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 00:01:41 +0600
Subject: [PATCH 1192/1515] [dfb] Extract formats

---
 youtube_dl/extractor/dfb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index f1df9b9a0..263532cc6 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -37,13 +37,13 @@ class DFBIE(InfoExtractor):
             self._proto_relative_url(video_info.find('url').text.strip()), display_id)
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
+        formats = self._extract_f4m_formats(manifest_url, display_id)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': video_info.find('title').text,
-            'url': manifest_url,
-            'ext': 'flv',
             'thumbnail': self._og_search_thumbnail(webpage),
             'upload_date': unified_strdate(video_info.find('time_date').text),
+            'formats': formats,
         }

From b062d94eefcc9327af5e60f3280cedaff9e40ac8 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 03:08:36 +0600
Subject: [PATCH 1193/1515] [README.md] Clarify authentication with .netrc file

---
 README.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/README.md b/README.md
index 9779c2058..a2cc89cdb 100644
--- a/README.md
+++ b/README.md
@@ -238,6 +238,26 @@ which means you can modify it, redistribute it or use it however you like.
 
 You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
 
+### Authentication with `.netrc` file ###
+
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in shell command history. You can achieve this using [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+```
+touch $HOME/.netrc
+chmod a-rwx,u+rw $HOME/.netrc
+```
+After that you can add credentials for extractor in the following format, where *extractor* is the name of extractor in lowercase:
+```
+machine <extractor> login <login> password <password>
+```
+For example:
+```
+machine youtube login myaccount@gmail.com password my_youtube_password
+machine twitch login my_twitch_account_name password my_twitch_password
+```
+To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or to place it in [configuration file](#configuration).
+
+On Windows you may also need to setup `%HOME%` environment variable manually.
+
 # OUTPUT TEMPLATE
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:

From 2af0f87c8b56567e0254aae7a1ccbedb04413b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 23:32:52 +0600
Subject: [PATCH 1194/1515] [prosiebensat1] Fix extraction (Closes #6215)

---
 youtube_dl/extractor/prosiebensat1.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 536a42dc8..22efa903f 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,8 +9,9 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
-    unified_strdate,
+    fix_xml_ampersands,
     int_or_none,
+    unified_strdate,
 )
 
 
@@ -208,7 +209,7 @@ class ProSiebenSat1IE(InfoExtractor):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'prosieben'
-        client_name = 'kolibri-1.12.6'
+        client_name = 'kolibri-2.0.19-splec4'
         client_location = url
 
         videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
@@ -275,8 +276,9 @@ class ProSiebenSat1IE(InfoExtractor):
 
         for source in urls_sources:
             protocol = source['protocol']
+            source_url = source['url']
             if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source['url'])
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
                 if not mobj:
                     continue
                 path = mobj.group('path')
@@ -293,9 +295,18 @@ class ProSiebenSat1IE(InfoExtractor):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
+            elif 'f4mgenerator' in source_url:
+                manifest = self._download_xml(
+                    source_url, clip_id, 'Downloading generated f4m manifest',
+                    transform_source=lambda s: fix_xml_ampersands(s).strip())
+                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
+                    manifest_url = media.get('href')
+                    if manifest_url:
+                        formats.extend(self._extract_f4m_formats(
+                            manifest_url, clip_id, f4m_id='hds'))
             else:
                 formats.append({
-                    'url': source['url'],
+                    'url': source_url,
                     'vbr': fix_bitrate(source['bitrate']),
                 })
 

From 97f4aecfc1c5fd446e1d3edd37e49aafe246fe0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:14:08 +0600
Subject: [PATCH 1195/1515] [extractor/common] Handle malformed f4m manifests

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3a396c0b0..f8a5ecced 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -28,6 +28,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
+    fix_xml_ampersands,
     float_or_none,
     int_or_none,
     RegexNotFoundError,
@@ -837,7 +838,10 @@ class InfoExtractor(object):
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
-            'Unable to download f4m manifest')
+            'Unable to download f4m manifest',
+            # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+            # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
+            transform_source=lambda s: fix_xml_ampersands(s).strip())
 
         formats = []
         manifest_version = '1.0'

From cc357c4db8112ff6736a227b47fb9527d327797f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:14:52 +0600
Subject: [PATCH 1196/1515] [extractor/common] Properly handle full URLs

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f8a5ecced..78e5cf8d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -851,8 +851,10 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
-                                (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                media_url = media_el.attrib.get('href') or media_el.attrib['url']
+                manifest_url = (
+                    media_url if media_url.startswith('http://') or media_url.startswith('https://')
+                    else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From 70f0f5a8ca53d4426fc079b3ab46e9d4a8e81ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:15:15 +0600
Subject: [PATCH 1197/1515] [extractor/common] Recursively extract child f4m
 manifests

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 78e5cf8d0..e3c610aa4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,6 +27,7 @@ from ..utils import (
     bug_reports_message,
     clean_html,
     compiled_regex_type,
+    determine_ext,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
@@ -855,6 +856,13 @@ class InfoExtractor(object):
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+                # If media_url is itself a f4m manifest do the recursive extraction
+                # since bitrates in parent manifest (this one) and media_url manifest
+                # may differ leading to inability to resolve the format by requested
+                # bitrate in f4m downloader
+                if determine_ext(manifest_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From f01f731107010e0c10fc94782daa7a3ba543e92a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:15:47 +0600
Subject: [PATCH 1198/1515] [prosiebensat1] Use generic f4m manifest extraction

---
 youtube_dl/extractor/prosiebensat1.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 22efa903f..2f9d95800 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
-    fix_xml_ampersands,
+    determine_ext,
     int_or_none,
     unified_strdate,
 )
@@ -295,15 +295,8 @@ class ProSiebenSat1IE(InfoExtractor):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
-            elif 'f4mgenerator' in source_url:
-                manifest = self._download_xml(
-                    source_url, clip_id, 'Downloading generated f4m manifest',
-                    transform_source=lambda s: fix_xml_ampersands(s).strip())
-                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
-                    manifest_url = media.get('href')
-                    if manifest_url:
-                        formats.extend(self._extract_f4m_formats(
-                            manifest_url, clip_id, f4m_id='hds'))
+            elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
+                formats.extend(self._extract_f4m_formats(source_url, clip_id))
             else:
                 formats.append({
                     'url': source_url,

From 31c746e5dc46491f997eca757c5e35842f04cb59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:25:33 +0600
Subject: [PATCH 1199/1515] [extractor/common] Keep going in some media_url is
 missing

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e3c610aa4..271bf8596 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -852,7 +852,9 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                media_url = media_el.attrib.get('href') or media_el.attrib['url']
+                media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
+                if not media_url:
+                    continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))

From ab9b890b524a49a9ffa4c8ac7243cd8afc15d270 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 02:23:07 +0600
Subject: [PATCH 1200/1515] [prosiebensat1] Clarify test purpose

---
 youtube_dl/extractor/prosiebensat1.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 2f9d95800..fec008ce7 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -22,6 +22,11 @@ class ProSiebenSat1IE(InfoExtractor):
 
     _TESTS = [
         {
+            # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242
+            # in response to fixing https://github.com/rg3/youtube-dl/issues/6215:
+            # - malformed f4m manifest support
+            # - proper handling of URLs starting with `https?://` in 2.0 manifests
+            # - recursive child f4m manifests extraction
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',

From e37c932fca29d93af77b7a47cccc9bb8578e3163 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Wed, 15 Jul 2015 15:13:56 -0500
Subject: [PATCH 1201/1515] compat_urllib_parse_unquote: crash fix: only decode
 valid hex

on python 2 the following has a { "crash_rate": "100%" } of the time
as it tries to parse '" ' as hex.
---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c3783337a..1f4ccf443 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -94,6 +94,8 @@ except ImportError:
             try:
                 if not item:
                     raise ValueError
+                if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]):
+                    raise ValueError
                 pct_sequence += item[:2].decode('hex')
                 rest = item[2:]
                 if not rest:

From 45eedbe58c8ab6344f11f1e1376d01648c1967ee Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Wed, 15 Jul 2015 15:30:47 -0500
Subject: [PATCH 1202/1515] Generic: use compat_urllib_parse_unquote to prevent
 utf8 mangling of the entire page in python 2.

-requires- fixed compat_urllib_parse_unquote

example - the following will save with a mangled playlist title,
 instead of the kanji for 'tsunami'. This affects all utf8encoded
 urls as well

youtube-dl -f18 -o '%(playlist_title)s-%(title)s.%(ext)s' \
  https://gist.githubusercontent.com/atomicdryad/fcb97465e6060fc519e1/raw/61c14c1e3a4985471dcf56c281d24d7e781a4e0e/tsunami.html
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 392ad3648..fc1bf2b6e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1115,7 +1115,7 @@ class GenericIE(InfoExtractor):
         # Sometimes embedded video player is hidden behind percent encoding
         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
         # Unescaping the whole page allows to handle those cases in a generic way
-        webpage = compat_urllib_parse.unquote(webpage)
+        webpage = compat_urllib_parse_unquote(webpage)
 
         # it's tempting to parse this further, but you would
         # have to take into account all the variations like

From e118031ef827e851e537daa5b439cf5c249ca88d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:21:04 +0600
Subject: [PATCH 1203/1515] [npo] Extend _VALID_URL to support ntr.nl (Closes
 #6248)

---
 youtube_dl/extractor/npo.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 62d12b7a6..1c823ec7f 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -37,8 +37,9 @@ class NPOBaseIE(InfoExtractor):
 
 
 class NPOIE(NPOBaseIE):
-    IE_NAME = 'npo.nl'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/(?!live|radio)[^/]+/[^/]+/(?P<id>[^/?]+)'
+    IE_NAME = 'npo'
+    IE_DESC = 'npo.nl and ntr.nl'
+    _VALID_URL = r'https?://(?:www\.)?(?:npo|ntr)\.nl/(?!live|radio)(?:[^/]+/){2,}(?P<id>[^/?#]+)'
 
     _TESTS = [
         {
@@ -100,6 +101,18 @@ class NPOIE(NPOBaseIE):
                 'title': 'Hoe gaat Europa verder na Parijs?',
             },
         },
+        {
+            'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
+            'md5': '01c6a2841675995da1f0cf776f03a9c3',
+            'info_dict': {
+                'id': 'VPWON_1233944',
+                'ext': 'm4v',
+                'title': 'Aap, poot, pies',
+                'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
+                'upload_date': '20150508',
+                'duration': 599,
+            },
+        }
     ]
 
     def _real_extract(self, url):

From 525daedd5a092b0f5329952eee99a7dac5537433 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:54:43 +0600
Subject: [PATCH 1204/1515] [npo] Add support for omroepwnl fragments

---
 youtube_dl/extractor/npo.py | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 1c823ec7f..a5162c0c6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,12 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -39,7 +45,16 @@ class NPOBaseIE(InfoExtractor):
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl and ntr.nl'
-    _VALID_URL = r'https?://(?:www\.)?(?:npo|ntr)\.nl/(?!live|radio)(?:[^/]+/){2,}(?P<id>[^/?#]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                            ntr\.nl/(?:[^/]+/){2,}|
+                            omroepwnl\.nl/video/fragment/[^/]+__
+                        )
+                        (?P<id>[^/?#]+)
+                '''
 
     _TESTS = [
         {
@@ -112,6 +127,18 @@ class NPOIE(NPOBaseIE):
                 'upload_date': '20150508',
                 'duration': 599,
             },
+        },
+        {
+            'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
+            'md5': 'd30cd8417b8b9bca1fdff27428860d08',
+            'info_dict': {
+                'id': 'POW_00996502',
+                'ext': 'm4v',
+                'title': '''"Dit is wel een 'landslide'..."''',
+                'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
+                'upload_date': '20150508',
+                'duration': 462,
+            },
         }
     ]
 
@@ -127,6 +154,11 @@ class NPOIE(NPOBaseIE):
             transform_source=strip_jsonp,
         )
 
+        # For some videos actual video id (prid) is different (e.g. for
+        # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698
+        # video id is POMS_WNL_853698 but prid is POW_00996502)
+        video_id = metadata.get('prid') or video_id
+
         token = self._get_token(video_id)
 
         formats = []

From 50ea2bb20d3a3e219910e87b8b30fc79ce534595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:56:57 +0600
Subject: [PATCH 1205/1515] [npo] Update test

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index a5162c0c6..cf6a388e5 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -87,7 +87,7 @@ class NPOIE(NPOBaseIE):
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
                 'title': 'Tegenlicht',
-                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
                 'duration': 3000,
             },

From 03f32a7eadf9d832aef55673edf38023a8daff95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:14:38 +0600
Subject: [PATCH 1206/1515] [wnl] Add extractor for omroepwnl playlists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 45 +++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f4f23521..1d55275dc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -390,6 +390,7 @@ from .npo import (
     NPORadioIE,
     NPORadioFragmentIE,
     TegenlichtVproIE,
+    WNLIE
 )
 from .nrk import (
     NRKIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index cf6a388e5..c6bf7619d 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -46,12 +46,15 @@ class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl and ntr.nl'
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        (?:
-                            npo\.nl/(?!live|radio)(?:[^/]+/){2}|
-                            ntr\.nl/(?:[^/]+/){2,}|
-                            omroepwnl\.nl/video/fragment/[^/]+__
+                    (?:
+                        npo:|
+                        https?://
+                            (?:www\.)?
+                            (?:
+                                npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                                ntr\.nl/(?:[^/]+/){2,}|
+                                omroepwnl\.nl/video/fragment/[^/]+__
+                            )
                         )
                         (?P<id>[^/?#]+)
                 '''
@@ -426,3 +429,33 @@ class TegenlichtVproIE(NPOIE):
         info_page = self._download_json(
             'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
         return self._get_info(info_page['mid'])
+
+
+class WNLIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'
+
+    _TEST = {
+        'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515',
+        'info_dict': {
+            'id': 'vandaag-de-dag-6-mei',
+            'title': 'Vandaag de Dag 6 mei',
+        },
+        'playlist_count': 4,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id, 'NPO')
+            for video_id, part in re.findall(
+                r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>(Deel \d+)', webpage)
+        ]
+
+        playlist_title = self._html_search_regex(
+            r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>',
+            webpage, 'playlist title')
+
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 611ac379bb466267aded6726f9c85e79b08168c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:34:24 +0600
Subject: [PATCH 1207/1515] [vpro] Fix extraction and add support for vpro
 playlists

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/npo.py      | 35 +++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d55275dc..06f21064b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -389,7 +389,7 @@ from .npo import (
     NPOLiveIE,
     NPORadioIE,
     NPORadioFragmentIE,
-    TegenlichtVproIE,
+    VPROIE,
     WNLIE
 )
 from .nrk import (
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c6bf7619d..28d5c90b3 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -404,9 +404,8 @@ class NPORadioFragmentIE(InfoExtractor):
         }
 
 
-class TegenlichtVproIE(NPOIE):
-    IE_NAME = 'tegenlicht.vpro.nl'
-    _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
+class VPROIE(NPOIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
 
     _TESTS = [
         {
@@ -416,19 +415,35 @@ class TegenlichtVproIE(NPOIE):
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
                 'title': 'Tegenlicht',
-                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },
         },
+        {
+            'url': 'http://www.vpro.nl/programmas/2doc/2015/sergio-herman.html',
+            'info_dict': {
+                'id': 'sergio-herman',
+                'title': 'Sergio Herman: Fucking perfect',
+            },
+            'playlist_count': 2,
+        }
     ]
 
     def _real_extract(self, url):
-        name = url_basename(url)
-        webpage = self._download_webpage(url, name)
-        urn = self._html_search_meta('mediaurn', webpage)
-        info_page = self._download_json(
-            'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
-        return self._get_info(info_page['mid'])
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id, 'NPO')
+            for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
+        ]
+
+        playlist_title = self._search_regex(
+            r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>',
+            webpage, 'playlist title', default=None) or self._og_search_title(webpage)
+
+        return self.playlist_result(entries, playlist_id, playlist_title)
 
 
 class WNLIE(InfoExtractor):

From 5ba761eb854d6e415b3ab542293cb31c073dc0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:39:22 +0600
Subject: [PATCH 1208/1515] [npo] Prefer aflevering_titel over titel

---
 youtube_dl/extractor/npo.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 28d5c90b3..91adb23f0 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -234,7 +234,9 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            'title': metadata['titel'],
+            # prefer aflevering_titel if any since titel may be too generic, e.g.
+            # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
+            'title': metadata.get('aflevering_titel') or metadata['titel'],
             'description': metadata['info'],
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
@@ -414,7 +416,7 @@ class VPROIE(NPOIE):
             'info_dict': {
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
-                'title': 'Tegenlicht',
+                'title': 'De toekomst komt uit Afrika',
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },

From 536b0700b03f0b29a1025be0b7753253bd627d6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:40:04 +0600
Subject: [PATCH 1209/1515] [npo] Allow missing description

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 91adb23f0..e733d96f6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -237,7 +237,7 @@ class NPOIE(NPOBaseIE):
             # prefer aflevering_titel if any since titel may be too generic, e.g.
             # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
             'title': metadata.get('aflevering_titel') or metadata['titel'],
-            'description': metadata['info'],
+            'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),

From 574f42d79a8596ceda681b205e19e766e7bab046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:53:12 +0600
Subject: [PATCH 1210/1515] [vpro] Improve playlist extraction

---
 youtube_dl/extractor/npo.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index e733d96f6..583ed3e14 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -428,6 +428,15 @@ class VPROIE(NPOIE):
                 'title': 'Sergio Herman: Fucking perfect',
             },
             'playlist_count': 2,
+        },
+        {
+            # playlist with youtube embed
+            'url': 'http://www.vpro.nl/programmas/2doc/2015/education-education.html',
+            'info_dict': {
+                'id': 'education-education',
+                'title': '2Doc',
+            },
+            'playlist_count': 2,
         }
     ]
 
@@ -437,7 +446,7 @@ class VPROIE(NPOIE):
         webpage = self._download_webpage(url, playlist_id)
 
         entries = [
-            self.url_result('npo:%s' % video_id, 'NPO')
+            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
             for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
         ]
 

From 1540119723e7e2195a47d659993a6a3bcc02d3e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:54:05 +0600
Subject: [PATCH 1211/1515] [npo] Remove unused imports

---
 youtube_dl/extractor/npo.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 583ed3e14..f5ffe1231 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -3,17 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
     qualities,
     strip_jsonp,
     unified_strdate,
-    url_basename,
 )
 
 

From 23fc384f2c3cf9afd41fd7e033fe0823d0fc5fa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 01:28:52 +0600
Subject: [PATCH 1212/1515] [npo] Compound title

---
 youtube_dl/extractor/npo.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index f5ffe1231..0c2d02c10 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -72,7 +72,7 @@ class NPOIE(NPOBaseIE):
             'info_dict': {
                 'id': 'VARA_101191800',
                 'ext': 'm4v',
-                'title': 'De Mega Mike & Mega Thomas show',
+                'title': 'De Mega Mike & Mega Thomas show: The best of.',
                 'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
                 'upload_date': '20090227',
                 'duration': 2400,
@@ -84,7 +84,7 @@ class NPOIE(NPOBaseIE):
             'info_dict': {
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
-                'title': 'Tegenlicht',
+                'title': 'Tegenlicht: De toekomst komt uit Afrika',
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
                 'duration': 3000,
@@ -157,6 +157,13 @@ class NPOIE(NPOBaseIE):
         # video id is POMS_WNL_853698 but prid is POW_00996502)
         video_id = metadata.get('prid') or video_id
 
+        # titel is too generic in some cases so utilize aflevering_titel as well
+        # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html)
+        title = metadata['titel']
+        sub_title = metadata.get('aflevering_titel')
+        if sub_title and sub_title != title:
+            title += ': %s' % sub_title
+
         token = self._get_token(video_id)
 
         formats = []
@@ -229,9 +236,7 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            # prefer aflevering_titel if any since titel may be too generic, e.g.
-            # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
-            'title': metadata.get('aflevering_titel') or metadata['titel'],
+            'title': title,
             'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),

From a38436e8898b6eca29d9279346e6e2136ec0bc8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Jul 2015 12:02:49 +0800
Subject: [PATCH 1213/1515] [extractor/common] Add 'transform_source' parameter
 to _extract_f4m_formats()

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 271bf8596..5a2d0d995 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -836,13 +836,14 @@ class InfoExtractor(object):
         self.to_screen(msg)
         time.sleep(timeout)
 
-    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
+    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
+                             transform_source=lambda s: fix_xml_ampersands(s).strip()):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
-            transform_source=lambda s: fix_xml_ampersands(s).strip())
+            transform_source=transform_source)
 
         formats = []
         manifest_version = '1.0'

From c9c854cea7fa5992356dee5eab0d3615b4d40dc6 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:31:29 -0500
Subject: [PATCH 1214/1515] replace old compat_urllib_parse_unquote with
 backport from python3's function

  * required unquote_to_bytes function ported as well
    (uses .decode('hex') instead of dynamically populated _hextobyte global)
  * required implicit conversion to bytes and/or unicode in places due to
    differing type assumptions in p3
---
 youtube_dl/compat.py | 75 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1f4ccf443..2fd2278aa 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,10 +74,81 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+from pprint import (pprint, pformat)
+
+
+def dprint(fmt):
+    sys.stderr.write(pformat(fmt) + "\n")
+
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
-    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+    def compat_urllib_parse_unquote_to_bytes(string):
+        """unquote_to_bytes('abc%20def') -> b'abc def'."""
+        # Note: strings are encoded as UTF-8. This is only an issue if it contains
+        # unescaped non-ASCII characters, which URIs should not.
+        if not string:
+            # Is it a string-like object?
+            string.split
+            return b''
+        if isinstance(string, str):
+            string = string.encode('utf-8')
+            # string = encode('utf-8')
+
+        # python3 -> 2: must implicitly convert to bits
+        bits = bytes(string).split(b'%')
+
+        if len(bits) == 1:
+            return string
+        res = [bits[0]]
+        append = res.append
+
+        for item in bits[1:]:
+            try:
+                append(item[:2].decode('hex'))
+                append(item[2:])
+            except:
+                append(b'%')
+                append(item)
+        return b''.join(res)
+
+    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
+
+    def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+        """Replace %xx escapes by their single-character equivalent. The optional
+        encoding and errors parameters specify how to decode percent-encoded
+        sequences into Unicode characters, as accepted by the bytes.decode()
+        method.
+        By default, percent-encoded sequences are decoded with UTF-8, and invalid
+        sequences are replaced by a placeholder character.
+
+        unquote('abc%20def') -> 'abc def'.
+        """
+
+        if '%' not in string:
+            string.split
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+
+        bits = compat_urllib_parse_asciire.split(string)
+        res = [bits[0]]
+        append = res.append
+        for i in range(1, len(bits), 2):
+            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
+            foo = foo.decode(encoding, errors)
+            append(foo)
+
+            if bits[i + 1]:
+                bar = bits[i + 1]
+                if not isinstance(bar, unicode):
+                    bar = bar.decode('utf-8')
+                append(bar)
+        return ''.join(res)
+
+    def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         if string == '':
             return string
         res = string.split('%')
@@ -114,6 +185,8 @@ except ImportError:
             string += pct_sequence.decode(encoding, errors)
         return string
 
+    compat_urllib_parse_unquote = new_compat_urllib_parse_unquote
+
 try:
     compat_str = unicode  # Python 2
 except NameError:

From 851229a01f34129286a57d46f8a27b9bb5fd9a6b Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:49:55 -0500
Subject: [PATCH 1215/1515] remove debugprint

---
 youtube_dl/compat.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2fd2278aa..554e3d5db 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,12 +74,6 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
-from pprint import (pprint, pformat)
-
-
-def dprint(fmt):
-    sys.stderr.write(pformat(fmt) + "\n")
-
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:

From a0f28f90fa277d9c00f0305624dea36a20b8066e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:50:43 -0500
Subject: [PATCH 1216/1515] remove kebab

---
 youtube_dl/compat.py | 41 +----------------------------------------
 1 file changed, 1 insertion(+), 40 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 554e3d5db..de9ba2c14 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -108,7 +108,7 @@ except ImportError:
 
     compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
 
-    def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         """Replace %xx escapes by their single-character equivalent. The optional
         encoding and errors parameters specify how to decode percent-encoded
         sequences into Unicode characters, as accepted by the bytes.decode()
@@ -142,45 +142,6 @@ except ImportError:
                 append(bar)
         return ''.join(res)
 
-    def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
-        if string == '':
-            return string
-        res = string.split('%')
-        if len(res) == 1:
-            return string
-        if encoding is None:
-            encoding = 'utf-8'
-        if errors is None:
-            errors = 'replace'
-        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
-        pct_sequence = b''
-        string = res[0]
-        for item in res[1:]:
-            try:
-                if not item:
-                    raise ValueError
-                if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]):
-                    raise ValueError
-                pct_sequence += item[:2].decode('hex')
-                rest = item[2:]
-                if not rest:
-                    # This segment was just a single percent-encoded character.
-                    # May be part of a sequence of code units, so delay decoding.
-                    # (Stored in pct_sequence).
-                    continue
-            except ValueError:
-                rest = '%' + item
-            # Encountered non-percent-encoded characters. Flush the current
-            # pct_sequence.
-            string += pct_sequence.decode(encoding, errors) + rest
-            pct_sequence = b''
-        if pct_sequence:
-            # Flush the final pct_sequence
-            string += pct_sequence.decode(encoding, errors)
-        return string
-
-    compat_urllib_parse_unquote = new_compat_urllib_parse_unquote
-
 try:
     compat_str = unicode  # Python 2
 except NameError:

From 36da48798a28b8261d2f39f73f2522651d58a364 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:27:50 -0500
Subject: [PATCH 1217/1515] handle titles and captions set to ''

---
 youtube_dl/extractor/bbc.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 471d865d2..c910eb55a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -497,11 +497,13 @@ class BBCNewsIE(BBCCoUkIE):
             programme_id = jent.get('externalId')
             xml_url = jent.get('href')
 
-            title = jent.get('caption',list_title)
+            title = jent.get('caption','')
+            if title == '':
+               title = list_title
 
             duration = parse_duration(jent.get('duration'))
             description = list_title
-            if jent.get('caption'):
+            if jent.get('caption', '') != '':
                description += ' - ' + jent.get('caption')
             thumbnail = None
             if jent.has_key('image'):

From a3bfddfa5ee33cf085b959536f1025c0aa53cc77 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:47:02 -0500
Subject: [PATCH 1218/1515] bbc.py: correct syntax

---
 youtube_dl/extractor/bbc.py | 106 ++++++++++++++++++------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c910eb55a..c8f285165 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -397,14 +397,14 @@ class BBCNewsIE(BBCCoUkIE):
             'title': 'Russia stages massive WW2 parade despite Western boycott',
         },
         'playlist_count': 2,
-    },{
+    }, {
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
             'title': 'Farnborough Airshow: Video highlights',
         },
         'playlist_count': 9,
-    },{
+    }, {
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'note': 'Video',
         'info_dict': {
@@ -419,7 +419,7 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
-    },{
+    }, {
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
         'note': 'Video',
         'info_dict': {
@@ -434,7 +434,7 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
-    },{
+    }, {
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
         'note': 'Video',
         'info_dict': {
@@ -459,88 +459,88 @@ class BBCNewsIE(BBCCoUkIE):
 
         pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
         if pubdate:
-           pubdate = pubdate.replace('-','')
+            pubdate = pubdate.replace('-', '')
 
         ret = []
         jsent = []
 
         # works with bbc.com/news/something-something-123456 articles
         jsent = map(
-           lambda m: self._parse_json(m,list_id),
-           re.findall(r"data-media-meta='({[^']+})'", webpage)
+            lambda m: self._parse_json(m, list_id),
+            re.findall(r"data-media-meta='({[^']+})'", webpage)
         )
 
         if len(jsent) == 0:
-           # http://www.bbc.com/news/video_and_audio/international
-           # and single-video articles
-           masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
-           if masset:
-              jmasset = self._parse_json(masset,list_id)
-              for key, val in jmasset.get('videos',{}).items():
-                  for skey, sval in val.items():
-                      sval['id'] = skey
-                      jsent.append(sval)
+            # http://www.bbc.com/news/video_and_audio/international
+            # and single-video articles
+            masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
+            if masset:
+                jmasset = self._parse_json(masset, list_id)
+                for key, val in jmasset.get('videos', {}).items():
+                    for skey, sval in val.items():
+                        sval['id'] = skey
+                        jsent.append(sval)
 
         if len(jsent) == 0:
-           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-           # in http://www.bbc.com/news/video_and_audio/international
-           # prone to breaking if entries have sourceFiles list
-           jsent = map(
-               lambda m: self._parse_json(m,list_id),
-               re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-           )          
+            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+            # in http://www.bbc.com/news/video_and_audio/international
+            # prone to breaking if entries have sourceFiles list
+            jsent = map(
+                lambda m: self._parse_json(m, list_id),
+                re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+            )
 
         if len(jsent) == 0:
-           raise ExtractorError('No video found', expected=True)
+            raise ExtractorError('No video found', expected=True)
 
         for jent in jsent:
             programme_id = jent.get('externalId')
             xml_url = jent.get('href')
 
-            title = jent.get('caption','')
+            title = jent.get('caption', '')
             if title == '':
-               title = list_title
+                title = list_title
 
             duration = parse_duration(jent.get('duration'))
             description = list_title
             if jent.get('caption', '') != '':
-               description += ' - ' + jent.get('caption')
+                description += ' - ' + jent.get('caption')
             thumbnail = None
-            if jent.has_key('image'):
-               thumbnail=jent['image'].get('href')
+            if jent.get('image') is not None:
+                thumbnail = jent['image'].get('href')
 
             formats = []
             subtitles = []
 
             if programme_id:
-               formats, subtitles = self._download_media_selector(programme_id)
-            elif jent.has_key('sourceFiles'):
-               # mediaselector not used at
-               # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
-               for key, val in jent['sourceFiles'].items():
-                  formats.append( {
-                     'ext': val.get('encoding'),
-                     'url': val.get('url'),
-                     'filesize': int(val.get('filesize')),
-                     'format_id': key
-                  } )
+                formats, subtitles = self._download_media_selector(programme_id)
+            elif jent.get('sourceFiles') is not None:
+                # mediaselector not used at
+                # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
+                for key, val in jent['sourceFiles'].items():
+                    formats.append({
+                        'ext': val.get('encoding'),
+                        'url': val.get('url'),
+                        'filesize': int(val.get('filesize')),
+                        'format_id': key
+                    })
             elif xml_url:
-               # Cheap fallback
-               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-               formats, subtitles = self._download_media_selector(programme_id)
+                # Cheap fallback
+                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+                formats, subtitles = self._download_media_selector(programme_id)
 
             if len(formats) == 0:
-               raise ExtractorError('unsupported json media entry.\n    '+str(jent)+'\n')
-               
+                raise ExtractorError('unsupported json media entry.\n    ' + str(jent) + '\n')
+
             self._sort_formats(formats)
 
-            id = jent.get('id') if programme_id == None else programme_id
-            if id == None:
-               id = 'NA'
+            id = jent.get('id') if programme_id is None else programme_id
+            if id is None:
+                id = 'NA'
 
-            ret.append( {
+            ret.append({
                 'id': id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
@@ -550,8 +550,8 @@ class BBCNewsIE(BBCCoUkIE):
                 'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
-            } )
+            })
 
         if len(ret) > 0:
-           return self.playlist_result(ret, list_id, list_title)
+            return self.playlist_result(ret, list_id, list_title)
         raise ExtractorError('No video found', expected=True)

From 9fefc88656eceac13604fd86dfb25dc736ed239a Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 07:24:07 -0500
Subject: [PATCH 1219/1515] fix TestCompat test_all_present

---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index de9ba2c14..8b4d0287c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -451,7 +451,9 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
+    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urlparse',

From 593b77064c51c411071e310578b542017b9b2ec8 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 09:45:49 -0500
Subject: [PATCH 1220/1515] Don't forget trailing '%'

---
 youtube_dl/compat.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8b4d0287c..9e506352f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -98,6 +98,9 @@ except ImportError:
         append = res.append
 
         for item in bits[1:]:
+            if item == '':
+                append(b'%')
+                continue
             try:
                 append(item[:2].decode('hex'))
                 append(item[2:])

From 4a632911443f0dbc2384fb82ade85382aeecc8dc Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 09:46:08 -0500
Subject: [PATCH 1221/1515] Add tests for compat_urllib_parse_unquote

---
 test/test_compat.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 1eb454e06..431e6bdf1 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
+    compat_urllib_parse_unquote,
 )
 
 
@@ -42,5 +43,24 @@ class TestCompat(unittest.TestCase):
             dir(youtube_dl.compat))) - set(['unicode_literals'])
         self.assertEqual(all_names, sorted(present_names))
 
+    def test_compat_urllib_parse_unquote(self):
+        test_strings = [
+            ['''''', ''''''],
+            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
+            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
+            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
+             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
+            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
+             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
+        ]
+        for test in test_strings:
+            strutf = test[0]
+            strurlenc = test[1]
+            strurldec = compat_urllib_parse_unquote(strurlenc)
+            self.assertEqual(strutf, strurldec)
+            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+
 if __name__ == '__main__':
     unittest.main()

From cabe001590914338524dff0c1edffbde4a1447ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:18:33 +0100
Subject: [PATCH 1222/1515] [howstuffwoks] fix _VALID_URL regex

---
 youtube_dl/extractor/howstuffworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index e97339121..f59393150 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(\d+-)*(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
             'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',

From f354385bf50e04b0e7e1defcc14264d1a66e0a07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:43:27 +0100
Subject: [PATCH 1223/1515] Add test for urls without a number

---
 youtube_dl/extractor/howstuffworks.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index f59393150..c9b6579a6 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -46,6 +46,17 @@ class HowStuffWorksIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        {
+            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
+            'info_dict': {
+                'id': '855410',
+                'ext': 'mp4',
+                'title': 'Stuff to Blow Your Mind',
+                'description': 'When it comes to optical illusions it’s pretty easy to jailbreak the operating system of your mind and load a new program onto it. Find out why your visual system willingly makes errors in interpretation.',
+                'display_id': 'optical-illusions',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        }
     ]
 
     def _real_extract(self, url):

From 1186e3f91ab940017be263c65c27b821992e58cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:45:53 +0100
Subject: [PATCH 1224/1515] do not capture a group

---
 youtube_dl/extractor/howstuffworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index c9b6579a6..cc931d571 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(\d+-)*(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
             'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',

From c4fe07c7afcc59dfcd5cfa4626ad37deb19e1033 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 17:07:55 +0100
Subject: [PATCH 1225/1515] match only the test url

---
 youtube_dl/extractor/howstuffworks.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index cc931d571..663e6632a 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -48,14 +48,7 @@ class HowStuffWorksIE(InfoExtractor):
         },
         {
             'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
-            'info_dict': {
-                'id': '855410',
-                'ext': 'mp4',
-                'title': 'Stuff to Blow Your Mind',
-                'description': 'When it comes to optical illusions it’s pretty easy to jailbreak the operating system of your mind and load a new program onto it. Find out why your visual system willingly makes errors in interpretation.',
-                'display_id': 'optical-illusions',
-                'thumbnail': 're:^https?://.*\.jpg$',
-            },
+            'only_matching': True,
         }
     ]
 

From 55139679261f8c2409ca150906a2693731452a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 22:58:13 +0600
Subject: [PATCH 1226/1515] [compat] Simplify and use latest cpython 3 code

---
 youtube_dl/compat.py | 41 +++++++++++++----------------------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e506352f..54ccf1d28 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -75,8 +75,13 @@ except ImportError:
     import BaseHTTPServer as compat_http_server
 
 try:
+    from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
-except ImportError:
+except ImportError:  # Python 2
+    # HACK: The following are the correct unquote_to_bytes and unquote
+    # implementations from cpython 3.4.3's stdlib. Python 2's version
+    # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
+
     def compat_urllib_parse_unquote_to_bytes(string):
         """unquote_to_bytes('abc%20def') -> b'abc def'."""
         # Note: strings are encoded as UTF-8. This is only an issue if it contains
@@ -85,32 +90,22 @@ except ImportError:
             # Is it a string-like object?
             string.split
             return b''
-        if isinstance(string, str):
+        if isinstance(string, unicode):
             string = string.encode('utf-8')
-            # string = encode('utf-8')
-
-        # python3 -> 2: must implicitly convert to bits
-        bits = bytes(string).split(b'%')
-
+        bits = string.split(b'%')
         if len(bits) == 1:
             return string
         res = [bits[0]]
         append = res.append
-
         for item in bits[1:]:
-            if item == '':
-                append(b'%')
-                continue
             try:
-                append(item[:2].decode('hex'))
+                append(compat_urllib_parse._hextochr[item[:2]])
                 append(item[2:])
-            except:
+            except KeyError:
                 append(b'%')
                 append(item)
         return b''.join(res)
 
-    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
-
     def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         """Replace %xx escapes by their single-character equivalent. The optional
         encoding and errors parameters specify how to decode percent-encoded
@@ -121,7 +116,6 @@ except ImportError:
 
         unquote('abc%20def') -> 'abc def'.
         """
-
         if '%' not in string:
             string.split
             return string
@@ -129,20 +123,12 @@ except ImportError:
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-
-        bits = compat_urllib_parse_asciire.split(string)
+        bits = compat_urllib_parse._asciire.split(string)
         res = [bits[0]]
         append = res.append
         for i in range(1, len(bits), 2):
-            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
-            foo = foo.decode(encoding, errors)
-            append(foo)
-
-            if bits[i + 1]:
-                bar = bits[i + 1]
-                if not isinstance(bar, unicode):
-                    bar = bar.decode('utf-8')
-                append(bar)
+            append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
+            append(bits[i + 1])
         return ''.join(res)
 
 try:
@@ -454,7 +440,6 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
-    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',

From 14309e1ddc476a7e2fc444a0443b2fc23186a385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 22:58:39 +0600
Subject: [PATCH 1227/1515] [test_compat] Make tests more idiomatic

---
 test/test_compat.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 431e6bdf1..2ffbc2c48 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -44,23 +44,22 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):
-        test_strings = [
-            ['''''', ''''''],
-            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
-            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
-            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
-%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
-             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
-%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
-            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
-             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
-        ]
-        for test in test_strings:
-            strutf = test[0]
-            strurlenc = test[1]
-            strurldec = compat_urllib_parse_unquote(strurlenc)
-            self.assertEqual(strutf, strurldec)
-            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+        self.assertEqual(compat_urllib_parse_unquote(''), '')
+        self.assertEqual(compat_urllib_parse_unquote('%'), '%')
+        self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
+        self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%')
+        self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
+        self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),
+            '''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
+            '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
 if __name__ == '__main__':
     unittest.main()

From d79febcd0619608aa100c17b0a2aefe9f4836d9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:09:56 +0600
Subject: [PATCH 1228/1515] [test_compat] Remove redundant test

---
 test/test_compat.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 2ffbc2c48..d816a9236 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -51,7 +51,6 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
         self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
         self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
-        self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波')
         self.assertEqual(
             compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
 %<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),

From aa99aa4e853966d4f01e280313e3aadf2ddf7ad6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:28:34 +0600
Subject: [PATCH 1229/1515] [compat] Add compat_urllib_parse_unquote_plus

---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 54ccf1d28..a3a2aef53 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -77,6 +77,7 @@ except ImportError:
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
+    from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
     # HACK: The following are the correct unquote_to_bytes and unquote
     # implementations from cpython 3.4.3's stdlib. Python 2's version
@@ -131,6 +132,15 @@ except ImportError:  # Python 2
             append(bits[i + 1])
         return ''.join(res)
 
+    def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
+        """Like unquote(), but also replace plus signs by spaces, as required for
+        unquoting HTML form values.
+
+        unquote_plus('%7e/abc+def') -> '~/abc def'
+        """
+        string = string.replace('+', ' ')
+        return compat_urllib_parse_unquote(string, encoding, errors)
+
 try:
     compat_str = unicode  # Python 2
 except NameError:
@@ -441,6 +451,7 @@ __all__ = [
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_plus',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',

From 8954e481404746c65ae1e82a6c5edec69114bfb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:31:23 +0600
Subject: [PATCH 1230/1515] [test_compat] Add tests for
 compat_urllib_parse_unquote_plus

---
 test/test_compat.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index d816a9236..c3ba8ad2e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -15,6 +15,7 @@ from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 
 
@@ -44,6 +45,8 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):
+        self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
         self.assertEqual(compat_urllib_parse_unquote(''), '')
         self.assertEqual(compat_urllib_parse_unquote('%'), '%')
         self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
@@ -60,5 +63,9 @@ class TestCompat(unittest.TestCase):
             compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
             '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
+    def test_compat_urllib_parse_unquote_plus(self):
+        self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
+
 if __name__ == '__main__':
     unittest.main()

From 4d08161ac23cfdf563b5d79b9f75cde6e6e57178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:32:43 +0600
Subject: [PATCH 1231/1515] [compat] Mention unquote_plus

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3a2aef53..e950a4688 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,7 +79,7 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
-    # HACK: The following are the correct unquote_to_bytes and unquote
+    # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
 

From b94b78971ce70a999584acab57d0a620a91bc2aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:35:14 +0600
Subject: [PATCH 1232/1515] [bet] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/bet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 26b934543..03dad4636 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     xpath_text,
     xpath_with_ns,
@@ -57,7 +57,7 @@ class BetIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        media_url = compat_urllib_parse.unquote(self._search_regex(
+        media_url = compat_urllib_parse_unquote(self._search_regex(
             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
             webpage, 'media URL'))
 

From 3e72f5f10e8dbac71e267ce10e791251a54e61ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:36:11 +0600
Subject: [PATCH 1233/1515] [ceskatelevize] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/ceskatelevize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 65f6be623..dda583680 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -88,7 +89,7 @@ class CeskaTelevizeIE(InfoExtractor):
         if playlist_url == 'error_region':
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlist_url))
+        req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
         playlist = self._download_json(req, video_id)

From a60cccbf9f76e7e42509bf1ed4e054643ecc348f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:36:58 +0600
Subject: [PATCH 1234/1515] [crunchyroll] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/crunchyroll.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 73f1e22ef..d1b6d7366 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -12,6 +12,7 @@ from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -254,7 +255,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             video_upload_date = unified_strdate(video_upload_date)
         video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
 
-        playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
+        playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = compat_urllib_request.Request(playerdata_url)
         playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')

From d3671b344f663c3aa82aea86de51a70f24511792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:37:23 +0600
Subject: [PATCH 1235/1515] [ehow] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/ehow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index 9cb1bf301..ba9685991 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from .common import InfoExtractor
 
@@ -26,7 +26,7 @@ class EHowIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
             r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
-        final_url = compat_urllib_parse.unquote(video_url)
+        final_url = compat_urllib_parse_unquote(video_url)
         uploader = self._html_search_meta('uploader', webpage)
         title = self._og_search_title(webpage).replace(' | eHow', '')
 

From d7011316d0bce88796a6d33e895fdafa231fd8ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:37:56 +0600
Subject: [PATCH 1236/1515] [facebook] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/facebook.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 82dc27bc6..e17bb9aea 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_http_client,
     compat_str,
     compat_urllib_error,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -136,7 +136,7 @@ class FacebookIE(InfoExtractor):
             else:
                 raise ExtractorError('Cannot parse data')
         data = dict(json.loads(m.group(1)))
-        params_raw = compat_urllib_parse.unquote(data['params'])
+        params_raw = compat_urllib_parse_unquote(data['params'])
         params = json.loads(params_raw)
         video_data = params['video_data'][0]
 

From 1f80e360fce9fc057847379ec06d0e635f697198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:38:30 +0600
Subject: [PATCH 1237/1515] [gamespot] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/gamespot.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 2d33fa7f5..b3f1bafcc 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -5,7 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -75,7 +75,7 @@ class GameSpotIE(InfoExtractor):
         return {
             'id': data_video['guid'],
             'display_id': page_id,
-            'title': compat_urllib_parse.unquote(data_video['title']),
+            'title': compat_urllib_parse_unquote(data_video['title']),
             'formats': formats,
             'description': self._html_search_meta('description', webpage),
             'thumbnail': self._og_search_thumbnail(webpage),

From f7e6f7fa2374864ba28fbe6840a0462b7c1033db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:39:32 +0600
Subject: [PATCH 1238/1515] [extractor/generic] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/generic.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fc1bf2b6e..a62287e50 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -8,7 +8,6 @@ import re
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
     compat_urlparse,
@@ -1369,7 +1368,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
         if mobj is not None:
-            return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
+            return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
 
         # Look for funnyordie embed
         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
@@ -1682,7 +1681,7 @@ class GenericIE(InfoExtractor):
         entries = []
         for video_url in found:
             video_url = compat_urlparse.urljoin(url, video_url)
-            video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
+            video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
 
             # Sometimes, jwplayer extraction will result in a YouTube URL
             if YoutubeIE.suitable(video_url):

From 8ee4ecb48dd7824f451c11c4cebb05b603a2da27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:39:54 +0600
Subject: [PATCH 1239/1515] [infoq] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/infoq.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 91a1b3ccb..71cfd12c5 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -4,7 +4,7 @@ import base64
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 
@@ -39,7 +39,7 @@ class InfoQIE(InfoExtractor):
         # Extract video URL
         encoded_id = self._search_regex(
             r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id')
-        real_id = compat_urllib_parse.unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
+        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
         playpath = 'mp4:' + real_id
 
         video_filename = playpath.split('/')[-1]

From 899a3e2f139a7b774d1a9483bb975e480e642c75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:40:35 +0600
Subject: [PATCH 1240/1515] [karaoketv] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/karaoketv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index e3b43ff8d..06daf5a89 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
     js_to_json,
 )
@@ -24,7 +24,7 @@ class KaraoketvIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         page_video_url = self._og_search_video_url(webpage, video_id)
-        config_json = compat_urllib_parse.unquote_plus(self._search_regex(
+        config_json = compat_urllib_parse_unquote_plus(self._search_regex(
             r'config=(.*)', page_video_url, 'configuration'))
 
         urls_info_json = self._download_json(

From 977a247a0639728146dd5c0d369b542e0a1af69b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:41:14 +0600
Subject: [PATCH 1241/1515] [malemotion] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/malemotion.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
index 0b85a59d1..92511a671 100644
--- a/youtube_dl/extractor/malemotion.py
+++ b/youtube_dl/extractor/malemotion.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class MalemotionIE(InfoExtractor):
@@ -24,7 +22,7 @@ class MalemotionIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_url = compat_urllib_parse.unquote(self._search_regex(
+        video_url = compat_urllib_parse_unquote(self._search_regex(
             r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)</title', webpage, 'title')

From c177bb3a50cde6035976603ba8094dd72cc310fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:41:47 +0600
Subject: [PATCH 1242/1515] [metacafe] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/metacafe.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 8bc333b02..6e2e73a51 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -155,7 +156,7 @@ class MetacafeIE(InfoExtractor):
         video_url = None
         mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
         if mobj is not None:
-            mediaURL = compat_urllib_parse.unquote(mobj.group(1))
+            mediaURL = compat_urllib_parse_unquote(mobj.group(1))
             video_ext = mediaURL[-3:]
 
             # Extract gdaKey if available

From 09b718c4398b2ce9964ec3ae5e329278d6211668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:42:11 +0600
Subject: [PATCH 1243/1515] [mitele] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mitele.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 7091f3335..852d72266 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -5,6 +5,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -48,7 +49,7 @@ class MiTeleIE(InfoExtractor):
             domain = 'http://' + domain
         info_url = compat_urlparse.urljoin(
             domain,
-            compat_urllib_parse.unquote(embed_data['flashvars']['host'])
+            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
         )
         info_el = self._download_xml(info_url, episode).find('./video/info')
 

From c2daf8dfa44b07179b272874c336d2435a2ff653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:42:43 +0600
Subject: [PATCH 1244/1515] [mixcloud] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mixcloud.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 425a4ccf1..d47aeceda 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -3,9 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
@@ -60,7 +58,7 @@ class MixcloudIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group(1)
         cloudcast_name = mobj.group(2)
-        track_id = compat_urllib_parse.unquote('-'.join((uploader, cloudcast_name)))
+        track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name)))
 
         webpage = self._download_webpage(url, track_id)
 

From e97bb3de838219a8c2800f3eb646313977503af2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:43:36 +0600
Subject: [PATCH 1245/1515] [mofosex] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mofosex.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index 2cec12d35..9bf99a54a 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -5,9 +5,9 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
 )
 
 
@@ -34,7 +34,7 @@ class MofosexIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, 'title')
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
+        video_url = compat_urllib_parse_unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[5].split('_')[:2]

From 9fd3bf04b7b160c1f24f9818689075f2570b8a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:44:38 +0600
Subject: [PATCH 1246/1515] [myvideo] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/myvideo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 5e754fcff..c96f472a3 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_ord,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -107,7 +108,7 @@ class MyVideoIE(InfoExtractor):
             if not a == '_encxml':
                 params[a] = b
             else:
-                encxml = compat_urllib_parse.unquote(b)
+                encxml = compat_urllib_parse_unquote(b)
         if not params.get('domain'):
             params['domain'] = 'www.myvideo.de'
         xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
@@ -135,7 +136,7 @@ class MyVideoIE(InfoExtractor):
         video_url = None
         mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
         if mobj:
-            video_url = compat_urllib_parse.unquote(mobj.group(1))
+            video_url = compat_urllib_parse_unquote(mobj.group(1))
             if 'myvideo2flash' in video_url:
                 self.report_warning(
                     'Rewriting URL to use unencrypted rtmp:// ...',
@@ -147,10 +148,10 @@ class MyVideoIE(InfoExtractor):
             mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
             if mobj is None:
                 raise ExtractorError('unable to extract url')
-            video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2))
+            video_url = compat_urllib_parse_unquote(mobj.group(1)) + compat_urllib_parse_unquote(mobj.group(2))
 
         video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
-        video_file = compat_urllib_parse.unquote(video_file)
+        video_file = compat_urllib_parse_unquote(video_file)
 
         if not video_file.endswith('f4m'):
             ppath, prefix = video_file.split('.')
@@ -159,7 +160,7 @@ class MyVideoIE(InfoExtractor):
             video_playpath = ''
 
         video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
-        video_swfobj = compat_urllib_parse.unquote(video_swfobj)
+        video_swfobj = compat_urllib_parse_unquote(video_swfobj)
 
         video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
                                               webpage, 'title')

From b78f5ec4c338e5e9379c1d7a2b0c99451e06897a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:00 +0600
Subject: [PATCH 1247/1515] [odnoklassniki] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/odnoklassniki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 6c7149fe3..215ffe87b 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     unified_strdate,
     int_or_none,
@@ -62,7 +62,7 @@ class OdnoklassnikiIE(InfoExtractor):
             metadata = self._parse_json(metadata, video_id)
         else:
             metadata = self._download_json(
-                compat_urllib_parse.unquote(flashvars['metadataUrl']),
+                compat_urllib_parse_unquote(flashvars['metadataUrl']),
                 video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']

From d54f1c74776a8d6c7cf07ae16ae67bd39ee51f93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:26 +0600
Subject: [PATCH 1248/1515] [openfilm] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/openfilm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
index 2249657eb..7e10bdf2c 100644
--- a/youtube_dl/extractor/openfilm.py
+++ b/youtube_dl/extractor/openfilm.py
@@ -5,7 +5,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote_plus,
     parse_age_limit,
     int_or_none,
 )
@@ -37,7 +37,7 @@ class OpenFilmIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        player = compat_urllib_parse.unquote_plus(
+        player = compat_urllib_parse_unquote_plus(
             self._og_search_video_url(webpage))
 
         video = json.loads(self._search_regex(

From 2ebbb6f1f771e5c81b103b6216073341e85cc1aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:55 +0600
Subject: [PATCH 1249/1515] [photobucket] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/photobucket.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index c66db3cdc..788411ccc 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -4,7 +4,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 
 
 class PhotobucketIE(InfoExtractor):
@@ -34,7 +34,7 @@ class PhotobucketIE(InfoExtractor):
         info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
                                        webpage, 'info json')
         info = json.loads(info_json)
-        url = compat_urllib_parse.unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
+        url = compat_urllib_parse_unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
         return {
             'id': video_id,
             'url': url,

From 388ad0c05c976a769ae9cf9d1f0e59526921b68c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:46:33 +0600
Subject: [PATCH 1250/1515] [playvid] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/playvid.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index c3e667e9e..2eb4fd96d 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -4,7 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     clean_html,
@@ -44,7 +45,7 @@ class PlayvidIE(InfoExtractor):
         flashvars = self._html_search_regex(
             r'flashvars="(.+?)"', webpage, 'flashvars')
 
-        infos = compat_urllib_parse.unquote(flashvars).split(r'&')
+        infos = compat_urllib_parse_unquote(flashvars).split(r'&')
         for info in infos:
             videovars_match = re.match(r'^video_vars\[(.+?)\]=(.+?)$', info)
             if videovars_match:
@@ -52,7 +53,7 @@ class PlayvidIE(InfoExtractor):
                 val = videovars_match.group(2)
 
                 if key == 'title':
-                    video_title = compat_urllib_parse.unquote_plus(val)
+                    video_title = compat_urllib_parse_unquote_plus(val)
                 if key == 'duration':
                     try:
                         duration = int(val)

From 605cbef65326b52ba9fdb994fc06798f15a3a1ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:47:23 +0600
Subject: [PATCH 1251/1515] [pornhub] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/pornhub.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 8172bc997..0b7886840 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -5,7 +5,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
@@ -69,7 +70,7 @@ class PornHubIE(InfoExtractor):
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:
-            thumbnail = compat_urllib_parse.unquote(thumbnail)
+            thumbnail = compat_urllib_parse_unquote(thumbnail)
 
         view_count = self._extract_count(
             r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
@@ -80,9 +81,9 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse.unquote_plus(
+            password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 

From 47af21e8f15be1ddb6195327719569d8723a669e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:47:51 +0600
Subject: [PATCH 1252/1515] [spankwire] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/spankwire.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index bff75d6b2..5fa6faf18 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
@@ -68,7 +68,7 @@ class SpankwireIE(InfoExtractor):
             webpage, 'comment count', fatal=False))
 
         video_urls = list(map(
-            compat_urllib_parse.unquote,
+            compat_urllib_parse_unquote,
             re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(

From 736f003f2e96b37a9554d77a26fdef6c0a38fd5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:49:14 +0600
Subject: [PATCH 1253/1515] [xbef] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xbef.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 80c48c37d..4ff99e5ca 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class XBefIE(InfoExtractor):
@@ -30,7 +28,7 @@ class XBefIE(InfoExtractor):
         config_url_enc = self._download_webpage(
             'http://xbef.com/Main/GetVideoURLEncoded/%s' % video_id, video_id,
             note='Retrieving config URL')
-        config_url = compat_urllib_parse.unquote(config_url_enc)
+        config_url = compat_urllib_parse_unquote(config_url_enc)
         config = self._download_xml(
             config_url, video_id, note='Retrieving config')
 

From 7dde5f6a8d10ba98b43f920c73248a8a7e73dc38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:49:38 +0600
Subject: [PATCH 1254/1515] [xnxx] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xnxx.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 79ed6c744..5a41f8ffa 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class XNXXIE(InfoExtractor):
@@ -26,7 +24,7 @@ class XNXXIE(InfoExtractor):
 
         video_url = self._search_regex(r'flv_url=(.*?)&amp;',
                                        webpage, 'video URL')
-        video_url = compat_urllib_parse.unquote(video_url)
+        video_url = compat_urllib_parse_unquote(video_url)
 
         video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM',
                                               webpage, 'title')

From ee8de13e14f0dee555c340ef8b9f9c15a6bac6dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:02 +0600
Subject: [PATCH 1255/1515] [xtube] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 1644f53c8..779e4f46a 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -5,7 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from ..utils import (
     parse_duration,
@@ -59,7 +59,7 @@ class XTubeIE(InfoExtractor):
         for format_id, video_url in re.findall(
                 r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
             fmt = {
-                'url': compat_urllib_parse.unquote(video_url),
+                'url': compat_urllib_parse_unquote(video_url),
                 'format_id': format_id,
             }
             m = re.search(r'^(?P<height>\d+)[pP]', format_id)
@@ -68,7 +68,7 @@ class XTubeIE(InfoExtractor):
             formats.append(fmt)
 
         if not formats:
-            video_url = compat_urllib_parse.unquote(self._search_regex(
+            video_url = compat_urllib_parse_unquote(self._search_regex(
                 r'flashvars\.video_url\s*=\s*"([^"]+)"',
                 webpage, 'video URL'))
             formats.append({'url': video_url})

From aa4789d632b71abb79561ecf6769258a66e158f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:25 +0600
Subject: [PATCH 1256/1515] [xvideos] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xvideos.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index d8415bed4..5dcf2fdd1 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -37,7 +37,7 @@ class XVideosIE(InfoExtractor):
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        video_url = compat_urllib_parse.unquote(
+        video_url = compat_urllib_parse_unquote(
             self._search_regex(r'flv_url=(.+?)&', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')

From db6c50f109e732f287cc0c61216cdd865a031ea3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:52 +0600
Subject: [PATCH 1257/1515] [ynet] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/ynet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 894678a23..869f3e819 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -5,7 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote_plus
 
 
 class YnetIE(InfoExtractor):
@@ -34,7 +34,7 @@ class YnetIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage))
+        content = compat_urllib_parse_unquote_plus(self._og_search_video_url(webpage))
         config = json.loads(self._search_regex(r'config=({.+?})$', content, 'video config'))
         f4m_url = config['clip']['url']
         title = self._og_search_title(webpage)

From 7fd002c0061f9f96eca2224553a2551748970e7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:51:57 +0600
Subject: [PATCH 1258/1515] [youtube] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/youtube.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3c629d38a..e7f5c7861 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -17,6 +17,8 @@ from ..compat import (
     compat_chr,
     compat_parse_qs,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
@@ -865,7 +867,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
         if mobj:
-            url = proto + '://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
+            url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
         video_id = self.extract_id(url)
 
         # Get video webpage
@@ -973,7 +975,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # uploader
         if 'author' not in video_info:
             raise ExtractorError('Unable to extract uploader name')
-        video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
+        video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
 
         # uploader_id
         video_uploader_id = None
@@ -1000,7 +1002,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning('unable to extract video thumbnail')
             video_thumbnail = None
         else:   # don't panic if we can't find it
-            video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
+            video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
 
         # upload date
         upload_date = self._html_search_meta(
@@ -1062,7 +1064,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning('unable to extract video duration')
             video_duration = None
         else:
-            video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
+            video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))
 
         # annotations
         video_annotations = None
@@ -1609,7 +1611,7 @@ class YoutubeSearchURLIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        query = compat_urllib_parse.unquote_plus(mobj.group('query'))
+        query = compat_urllib_parse_unquote_plus(mobj.group('query'))
 
         webpage = self._download_webpage(url, query)
         result_code = self._search_regex(

From c60e8cfaf7405e8b8075f06a62e7547adad8beba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:54:38 +0600
Subject: [PATCH 1259/1515] [ehow] Simplify

---
 youtube_dl/extractor/ehow.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index ba9685991..b1cd4f5d4 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-from ..compat import (
-    compat_urllib_parse_unquote,
-)
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 
 
 class EHowIE(InfoExtractor):

From 7bd42d0d963b543aae1f4b4e8c157dc3421189fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:56:27 +0600
Subject: [PATCH 1260/1515] [openfilm] Fix compat_urllib_parse_unquote_plus
 import

---
 youtube_dl/extractor/openfilm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
index 7e10bdf2c..d2ceedd01 100644
--- a/youtube_dl/extractor/openfilm.py
+++ b/youtube_dl/extractor/openfilm.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
     parse_iso8601,
-    compat_urllib_parse_unquote_plus,
     parse_age_limit,
     int_or_none,
 )

From 6b19647d5706b75b5e2d7da836495fe5174919f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 00:04:25 +0600
Subject: [PATCH 1261/1515] [veehd] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/veehd.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 346edf485..0d8d832cc 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -5,6 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -76,7 +77,7 @@ class VeeHDIE(InfoExtractor):
 
         if config_json:
             config = json.loads(config_json)
-            video_url = compat_urlparse.unquote(config['clip']['url'])
+            video_url = compat_urllib_parse_unquote(config['clip']['url'])
 
         if not video_url:
             video_url = self._html_search_regex(

From 3cc8b4c327585ff4dbb045d17d6f5c160df6fdb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 00:24:39 +0600
Subject: [PATCH 1262/1515] [compat] Fix missing _asciire on python 2.6

---
 youtube_dl/compat.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e950a4688..db0da5828 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,6 +79,8 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
+    _asciire = re.compile('([\x00-\x7f]+)') if sys.version_info < (2, 7) else compat_urllib_parse._asciire
+
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
@@ -124,7 +126,7 @@ except ImportError:  # Python 2
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-        bits = compat_urllib_parse._asciire.split(string)
+        bits = _asciire.split(string)
         res = [bits[0]]
         append = res.append
         for i in range(1, len(bits), 2):

From 9441f77faaf078aca2947b7613f61f46b6d0aea2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 01:45:36 +0600
Subject: [PATCH 1263/1515] [bliptv] Use xpath for extraction and fix test

---
 youtube_dl/extractor/bliptv.py | 51 +++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index fb56cd78d..cd06dbcea 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -14,6 +14,8 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     unescapeHTML,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
@@ -23,10 +25,10 @@ class BlipTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352',
-            'md5': 'c6934ad0b6acf2bd920720ec888eb812',
+            'md5': '80baf1ec5c3d2019037c1c707d676b9f',
             'info_dict': {
                 'id': '5779306',
-                'ext': 'mov',
+                'ext': 'm4v',
                 'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3',
                 'description': 'md5:9bc31f227219cde65e47eeec8d2dc596',
                 'timestamp': 1323138843,
@@ -128,35 +130,34 @@ class BlipTVIE(InfoExtractor):
 
         rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS')
 
-        def blip(s):
-            return '{http://blip.tv/dtd/blip/1.0}%s' % s
-
-        def media(s):
-            return '{http://search.yahoo.com/mrss/}%s' % s
-
-        def itunes(s):
-            return '{http://www.itunes.com/dtds/podcast-1.0.dtd}%s' % s
+        def _x(p):
+            return xpath_with_ns(p, {
+                'blip': 'http://blip.tv/dtd/blip/1.0',
+                'media': 'http://search.yahoo.com/mrss/',
+                'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
+            })
 
         item = rss.find('channel/item')
 
-        video_id = item.find(blip('item_id')).text
-        title = item.find('./title').text
-        description = clean_html(compat_str(item.find(blip('puredescription')).text))
-        timestamp = parse_iso8601(item.find(blip('datestamp')).text)
-        uploader = item.find(blip('user')).text
-        uploader_id = item.find(blip('userid')).text
-        duration = int(item.find(blip('runtime')).text)
-        media_thumbnail = item.find(media('thumbnail'))
-        thumbnail = media_thumbnail.get('url') if media_thumbnail is not None else item.find(itunes('image')).text
-        categories = [category.text for category in item.findall('category')]
+        video_id = xpath_text(item, _x('blip:item_id'), 'video id') or lookup_id
+        title = xpath_text(item, 'title', 'title', fatal=True)
+        description = clean_html(xpath_text(item, _x('blip:puredescription'), 'description'))
+        timestamp = parse_iso8601(xpath_text(item, _x('blip:datestamp'), 'timestamp'))
+        uploader = xpath_text(item, _x('blip:user'), 'uploader')
+        uploader_id = xpath_text(item, _x('blip:userid'), 'uploader id')
+        duration = int_or_none(xpath_text(item, _x('blip:runtime'), 'duration'))
+        media_thumbnail = item.find(_x('media:thumbnail'))
+        thumbnail = (media_thumbnail.get('url') if media_thumbnail is not None
+                     else xpath_text(item, 'image', 'thumbnail'))
+        categories = [category.text for category in item.findall('category') if category is not None]
 
         formats = []
         subtitles_urls = {}
 
-        media_group = item.find(media('group'))
-        for media_content in media_group.findall(media('content')):
+        media_group = item.find(_x('media:group'))
+        for media_content in media_group.findall(_x('media:content')):
             url = media_content.get('url')
-            role = media_content.get(blip('role'))
+            role = media_content.get(_x('blip:role'))
             msg = self._download_webpage(
                 url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url',
                 video_id, 'Resolving URL for %s' % role)
@@ -175,8 +176,8 @@ class BlipTVIE(InfoExtractor):
                     'url': real_url,
                     'format_id': role,
                     'format_note': media_type,
-                    'vcodec': media_content.get(blip('vcodec')) or 'none',
-                    'acodec': media_content.get(blip('acodec')),
+                    'vcodec': media_content.get(_x('blip:vcodec')) or 'none',
+                    'acodec': media_content.get(_x('blip:acodec')),
                     'filesize': media_content.get('filesize'),
                     'width': int_or_none(media_content.get('width')),
                     'height': int_or_none(media_content.get('height')),

From 530857182df72db028f959c0a58a9daf219d6b22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 01:45:54 +0600
Subject: [PATCH 1264/1515] [bliptv] Add test with missing duration

---
 youtube_dl/extractor/bliptv.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index cd06dbcea..a69ee482b 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -102,6 +102,20 @@ class BlipTVIE(InfoExtractor):
                 'vcodec': 'none',
             }
         },
+        {
+            # missing duration
+            'url': 'http://blip.tv/rss/flash/6700880',
+            'info_dict': {
+                'id': '6684191',
+                'ext': 'm4v',
+                'title': 'Cowboy Bebop: Gateway Shuffle Review',
+                'description': 'md5:3acc480c0f9ae157f5fe88547ecaf3f8',
+                'timestamp': 1386639757,
+                'upload_date': '20131210',
+                'uploader': 'sfdebris',
+                'uploader_id': '706520',
+            }
+        }
     ]
 
     @staticmethod

From 3c283a381e4f7a69bf57c3ea85aab3c85ce0e309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 02:43:18 +0600
Subject: [PATCH 1265/1515] [sbs] Simplify

---
 youtube_dl/extractor/sbs.py | 40 +++++++++++++++----------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index ab4d1c884..d6ee2d9e2 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -2,12 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import remove_end
 
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/(?:single/)?(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P<id>[0-9]+)'
 
     _TESTS = [{
         # Original URL is handled by the generic IE which finds the iframe:
@@ -17,43 +16,36 @@ class SBSIE(InfoExtractor):
         'info_dict': {
             'id': '320403011771',
             'ext': 'mp4',
-            'title': 'Dingo Conservation',
-            'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction',
+            'title': 'Dingo Conservation (The Feed)',
+            'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
             'thumbnail': 're:http://.*\.jpg',
+            'duration': 308,
         },
-        'add_ies': ['generic'],
     }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
         'only_matching': True,
+    }, {
+        'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # the video is in the following iframe
-        iframe_url = 'http://www.sbs.com.au/ondemand/video/single/' + video_id + '?context=web'
-        webpage = self._download_webpage(iframe_url, video_id)
+        webpage = self._download_webpage(
+            'http://www.sbs.com.au/ondemand/video/single/%s?context=web' % video_id, video_id)
 
-        player_params = self._search_regex(
-            r'(?s)(playerParams.+?releaseUrls.+?\n)',
-            webpage, 'playerParams')
-        player_params_js = self._search_regex(
-            r'({.*})',
-            player_params, 'player_param_js')
+        player_params = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+playerParams\s*=\s*({.+?});', webpage, 'playerParams'),
+            video_id)
 
-        player_params_json = self._parse_json(player_params_js, video_id)
-
-        theplatform_url = player_params_json.get('releaseUrls')['progressive'] or player_params_json.get('releaseUrls')['standard']
-
-        title = remove_end(self._og_search_title(webpage, default=video_id, fatal=False), ' (The Feed)')
-        description = self._html_search_meta('description', webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        urls = player_params['releaseUrls']
+        theplatform_url = (urls.get('progressive') or urls.get('standard') or
+                           urls.get('html') or player_params['relatedItemsURL'])
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
             'url': theplatform_url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
         }

From 4e1ad6e9a86e03a4309ed63914a08484aaff2784 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:13:45 +0600
Subject: [PATCH 1266/1515] [videomega] Extend _VALID_URL and improve
 extraction (Closes #6260)

---
 youtube_dl/extractor/videomega.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index eb309a7cd..d45f88646 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,17 +8,14 @@ from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://
-        (?:www\.)?videomega\.tv/
-        (?:iframe\.php|cdn\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
-        '''
+    _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
     _TEST = {
-        'url': 'http://videomega.tv/?ref=4GNA688SU99US886ANG4',
-        'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
-            'id': '4GNA688SU99US886ANG4',
+            'id': 'AOSQBJYKIDDIKYJBQSOA',
             'ext': 'mp4',
-            'title': 'BigBuckBunny_320x180',
+            'title': '1254207',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -26,15 +23,15 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
-        req = compat_urllib_request.Request(iframe_url)
+        req = compat_urllib_request.Request(url)
         req.add_header('Referer', url)
+        req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
 
         title = self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title')
+            r'<title>(.+?)</title>', webpage, 'title')
         title = re.sub(
-            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s?|\s?-\svideomega\.tv$)', '', title)
+            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
             r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
         video_url = self._search_regex(
@@ -46,6 +43,6 @@ class VideoMegaIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
-                'Referer': iframe_url,
+                'Referer': url,
             },
         }

From 8a37f53685c3a9227aad139c9faf13df8576abb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:25:10 +0600
Subject: [PATCH 1267/1515] [videomega] Revert iframe URL

---
 youtube_dl/extractor/videomega.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index d45f88646..42d42adb0 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -10,7 +10,7 @@ from ..compat import compat_urllib_request
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
     _TEST = {
-        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
             'id': 'AOSQBJYKIDDIKYJBQSOA',
@@ -23,7 +23,8 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
+        req = compat_urllib_request.Request(iframe_url)
         req.add_header('Referer', url)
         req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
@@ -43,6 +44,6 @@ class VideoMegaIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
-                'Referer': url,
+                'Referer': iframe_url,
             },
         }

From 4bdfef5a1874d99457cf5758c5d8e1cc0c487f08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:25:30 +0600
Subject: [PATCH 1268/1515] [videomega] Add tests

---
 youtube_dl/extractor/videomega.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 42d42adb0..b2ef637fa 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -9,7 +9,7 @@ from ..compat import compat_urllib_request
 
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
@@ -18,7 +18,13 @@ class VideoMegaIE(InfoExtractor):
             'title': '1254207',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomega.tv/view.php?ref=090051111052065112106089103052052103089106112065052111051090',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 4211e1941b39cb153505c64ab8fcb31a762674d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:27:09 +0600
Subject: [PATCH 1269/1515] [videomega] Add shortcut to _VALID_URL

---
 youtube_dl/extractor/videomega.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index b2ef637fa..78ff6310a 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',

From 1793d71db61acc9f6cceefbc63d7de2b558b7582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:18:03 +0600
Subject: [PATCH 1270/1515] [twitch:stream] Fix channel_id in different case
 (Closes #6263)

---
 youtube_dl/extractor/twitch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 92b6dc1b8..ab557a1a5 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -346,6 +346,8 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
+        channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
+
         access_token = self._download_json(
             '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
             'Downloading channel access token')

From 06966677343c4bbb51e8da3f0ed4d4985bdf54f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:27:45 +0600
Subject: [PATCH 1271/1515] [twitch:stream] Clarify channel_id reassignment
 rationale (#6263)

---
 youtube_dl/extractor/twitch.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index ab557a1a5..948c8ce39 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -346,6 +346,10 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
+        # Channel name may be typed if different case than the original channel name
+        # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
+        # an invalid m3u8 URL. Working around by use of original channel name from stream
+        # JSON and fallback to lowercase if it's not available.
         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
 
         access_token = self._download_json(

From f29ac588ddb5b36ebfe38e3a809a70cf0cb543a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:33:50 +0600
Subject: [PATCH 1272/1515] [nationalgeographics] Fix extraction (Closes #6262)

---
 youtube_dl/extractor/nationalgeographic.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index c18640c5a..f793b72f5 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -25,8 +25,11 @@ class NationalGeographicIE(InfoExtractor):
         name = url_basename(url)
 
         webpage = self._download_webpage(url, name)
-        feed_url = self._search_regex(r'data-feed-url="([^"]+)"', webpage, 'feed url')
-        guid = self._search_regex(r'data-video-guid="([^"]+)"', webpage, 'guid')
+        feed_url = self._search_regex(
+            r'data-feed-url="([^"]+)"', webpage, 'feed url')
+        guid = self._search_regex(
+            r'id="(?:videoPlayer|player-container)"[^>]+data-guid="([^"]+)"',
+            webpage, 'guid')
 
         feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
         content = feed.find('.//{http://search.yahoo.com/mrss/}content')

From 1b541d8d6ee5e3543253501d725af7ce4a55cad8 Mon Sep 17 00:00:00 2001
From: sceext <sceext@foxmail.com>
Date: Sat, 18 Jul 2015 12:29:35 +0800
Subject: [PATCH 1273/1515] [iqiyi] fix iqiyi (2015-07-17), update the md5 salt
 (enc_key) to iqiyi latest (2015-07-17) flash player.

---
 youtube_dl/extractor/iqiyi.py | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 0f6707d7c..ac5416953 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -212,20 +212,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        filename, _ = os.path.splitext(url_basename(swf_url))
-        enc_key_json = self._downloader.cache.load('iqiyi-enc-key', filename)
-        if enc_key_json is not None:
-            return enc_key_json[0]
-
-        req = self._request_webpage(
-            swf_url, video_id, note='download swf content')
-        cn = req.read()
-        cn = zlib.decompress(cn[8:])
-        pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
-        enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
-
-        self._downloader.cache.store('iqiyi-enc-key', filename, [enc_key])
-
+        enc_key = '8e29ab5666d041c3a1ea76e06dabdffb'
         return enc_key
 
     def _real_extract(self, url):

From b0bff54b08a44966d321e8de0ed08dc1dde6b96e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 18 Jul 2015 11:59:45 +0200
Subject: [PATCH 1274/1515] release 2015.07.18

---
 docs/supportedsites.md | 58 +++++++++++++++++++++++++++---------------
 youtube_dl/version.py  |  2 +-
 2 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0ca06c71d..a84878026 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,7 +28,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
- - **AppleDaily**
+ - **AppleDaily**: 臺灣蘋果日報
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -45,7 +45,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
- - **BaiduVideo**
+ - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
  - **Bandcamp**
@@ -106,7 +106,7 @@
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
- - **CtsNews**
+ - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
  - **dailymotion**
  - **dailymotion:playlist**
@@ -121,7 +121,7 @@
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
- - **DouyuTV**
+ - **DouyuTV**: 斗鱼
  - **dramafever**
  - **dramafever:series**
  - **DRBonanza**
@@ -222,7 +222,7 @@
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
  - **IPrima**
- - **iqiyi**
+ - **iqiyi**: 爱奇艺
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -243,9 +243,15 @@
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
+ - **kuwo:album**: 酷我音乐 - 专辑
+ - **kuwo:category**: 酷我音乐 - 分类
+ - **kuwo:chart**: 酷我音乐 - 排行榜
+ - **kuwo:mv**: 酷我音乐 - MV
+ - **kuwo:singer**: 酷我音乐 - 歌手
+ - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
- - **Letv**
+ - **Letv**: 乐视网
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
@@ -297,6 +303,7 @@
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
+ - **Myvi**
  - **myvideo**
  - **MyVidster**
  - **N-JOY**
@@ -312,11 +319,18 @@
  - **NDTV**
  - **NerdCubedFeed**
  - **Nerdist**
+ - **netease:album**: 网易云音乐 - 专辑
+ - **netease:djradio**: 网易云音乐 - 电台
+ - **netease:mv**: 网易云音乐 - MV
+ - **netease:playlist**: 网易云音乐 - 歌单
+ - **netease:program**: 网易云音乐 - 电台节目
+ - **netease:singer**: 网易云音乐 - 歌手
+ - **netease:song**: 网易云音乐
  - **Netzkino**
  - **Newgrounds**
  - **Newstube**
- - **NextMedia**
- - **NextMediaActionNews**
+ - **NextMedia**: 蘋果日報
+ - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
@@ -332,13 +346,14 @@
  - **Nowness**
  - **NowTV**
  - **nowvideo**: NowVideo
- - **npo.nl**
+ - **npo**: npo.nl and ntr.nl
+ - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
  - **NRK**
  - **NRKPlaylist**
- - **NRKTV**
+ - **NRKTV**: NRK TV and NRK Radio
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
@@ -382,11 +397,11 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
- - **qqmusic**
- - **qqmusic:album**
- - **qqmusic:playlist**
- - **qqmusic:singer**
- - **qqmusic:toplist**
+ - **qqmusic**: QQ音乐
+ - **qqmusic:album**: QQ音乐 - 专辑
+ - **qqmusic:playlist**: QQ音乐 - 歌单
+ - **qqmusic:singer**: QQ音乐 - 歌手
+ - **qqmusic:toplist**: QQ音乐 - 排行榜
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -395,6 +410,7 @@
  - **RadioJavan**
  - **Rai**
  - **RBMARadio**
+ - **RDS**: RDS.ca
  - **RedTube**
  - **Restudy**
  - **ReverbNation**
@@ -495,7 +511,6 @@
  - **TechTalks**
  - **techtv.mit.edu**
  - **ted**
- - **tegenlicht.vpro.nl**
  - **TeleBruxelles**
  - **telecinco.es**
  - **TeleMB**
@@ -551,7 +566,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
- - **UDNEmbed**
+ - **UDNEmbed**: 聯合影音
  - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
@@ -613,9 +628,11 @@
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
  - **WebOfStories**
+ - **WebOfStoriesPlaylist**
  - **Weibo**
  - **Wimp**
  - **Wistia**
+ - **WNL**
  - **WorldStarHipHop**
  - **wrzuta.pl**
  - **WSJ**: Wall Street Journal
@@ -628,18 +645,19 @@
  - **Xstream**
  - **XTube**
  - **XTubeUser**: XTube user profile
- - **Xuite**
+ - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
- - **Yam**
+ - **Yam**: 蕃薯藤yam天空部落
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
  - **YesJapan**
+ - **yinyuetai:video**: 音悦Tai
  - **Ynet**
  - **YouJizz**
- - **youku**
+ - **youku**: 优酷
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3364647ed..3ad7a2bc0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.07'
+__version__ = '2015.07.18'

From 74fe23ec35483c552f9c253be4c565546f78f001 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 16:35:28 +0600
Subject: [PATCH 1275/1515] [extractor/common] Style

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2d0d995..b9014fc23 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -996,7 +996,7 @@ class InfoExtractor(object):
     def _parse_smil_video(self, video, video_id, base, rtmp_count):
         src = video.get('src')
         if not src:
-            return ([], rtmp_count)
+            return [], rtmp_count
         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
         width = int_or_none(video.get('width'))
         height = int_or_none(video.get('height'))
@@ -1009,7 +1009,7 @@ class InfoExtractor(object):
                     proto = 'http'
         ext = video.get('ext')
         if proto == 'm3u8':
-            return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
+            return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
         elif proto == 'rtmp':
             rtmp_count += 1
             streamer = video.get('streamer') or base

From dc786d3db59a611cbcb716a476602719e74a34e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:22:25 +0600
Subject: [PATCH 1276/1515] [vk:uservideos] Improve extraction

---
 youtube_dl/extractor/vk.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 8f677cae3..93c5b18c9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -291,23 +291,32 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk.com:user-videos'
     IE_DESC = 'vk.com:All of a user\'s videos'
-    _VALID_URL = r'https?://vk\.com/videos(?P<id>[0-9]+)(?:m\?.*)?'
+    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
     _TEMPLATE_URL = 'https://vk.com/videos'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://vk.com/videos205387401',
         'info_dict': {
             'id': '205387401',
+            'title': "Tom Cruise's Videos",
         },
         'playlist_mincount': 4,
-    }
+    }, {
+        'url': 'http://vk.com/videos-77521',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
-        page = self._download_webpage(url, page_id)
-        video_ids = orderedSet(
-            m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
-        url_entries = [
+
+        webpage = self._download_webpage(url, page_id)
+
+        entries = [
             self.url_result(
                 'http://vk.com/video' + video_id, 'VK', video_id=video_id)
-            for video_id in video_ids]
-        return self.playlist_result(url_entries, page_id)
+            for video_id in set(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
+
+        title = unescapeHTML(self._search_regex(
+            r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',
+            webpage, 'title', default=page_id))
+
+        return self.playlist_result(entries, page_id, title)

From 1ecb5d1d83970d6295c655a37c99e390fb5d7dee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:23:33 +0600
Subject: [PATCH 1277/1515] [vk] Clarify extractor names

---
 youtube_dl/extractor/vk.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 93c5b18c9..ef94b6894 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -20,7 +20,8 @@ from ..utils import (
 
 
 class VKIE(InfoExtractor):
-    IE_NAME = 'vk.com'
+    IE_NAME = 'vk'
+    IE_DESC = 'VK'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
@@ -289,8 +290,8 @@ class VKIE(InfoExtractor):
 
 
 class VKUserVideosIE(InfoExtractor):
-    IE_NAME = 'vk.com:user-videos'
-    IE_DESC = 'vk.com:All of a user\'s videos'
+    IE_NAME = 'vk:uservideos'
+    IE_DESC = "VK - User's Videos"
     _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{

From c6b68648f45498bcacc71cc8f29696fa93259a7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:36:46 +0600
Subject: [PATCH 1278/1515] [bilibili] Show georestriction error

---
 youtube_dl/extractor/bilibili.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index bf60450c2..ecc17ebeb 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -41,8 +41,15 @@ class BiliBiliIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        if self._search_regex(r'(此视频不存在或被删除)', webpage, 'error message', default=None):
-            raise ExtractorError('The video does not exist or was deleted', expected=True)
+        if '(此视频不存在或被删除)' in webpage:
+            raise ExtractorError(
+                'The video does not exist or was deleted', expected=True)
+
+        if '>你没有权限浏览！ 由于版权相关问题 我们不对您所在的地区提供服务<' in webpage:
+            raise ExtractorError(
+                'The video is not available in your region due to copyright reasons',
+                expected=True)
+
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 

From fec73daaa30d006bc6c5d0483d255b7bc1c256b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 18:05:08 +0600
Subject: [PATCH 1279/1515] [vk:uservideos] Revert orderedSet

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ef94b6894..98bd3a141 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -314,7 +314,7 @@ class VKUserVideosIE(InfoExtractor):
         entries = [
             self.url_result(
                 'http://vk.com/video' + video_id, 'VK', video_id=video_id)
-            for video_id in set(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
+            for video_id in orderedSet(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
 
         title = unescapeHTML(self._search_regex(
             r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',

From 22603348aa0b3e02c520589dea092507a04ab06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 18:32:52 +0600
Subject: [PATCH 1280/1515] [compat] Fix _asciire

---
 youtube_dl/compat.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index db0da5828..0c57c7aeb 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,7 +79,8 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
-    _asciire = re.compile('([\x00-\x7f]+)') if sys.version_info < (2, 7) else compat_urllib_parse._asciire
+    _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
+                else re.compile('([\x00-\x7f]+)'))
 
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version

From 9ac09ed4deafc77cfd94e6a8827e4b0c47517b5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:13:10 +0600
Subject: [PATCH 1281/1515] [bliptv] Remove unused import

---
 youtube_dl/extractor/bliptv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index a69ee482b..c3296283d 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 
 from ..compat import (
-    compat_str,
     compat_urllib_request,
     compat_urlparse,
 )

From ee48b6a88f4b8f076699f539ee4c92389fa37643 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:15:20 +0600
Subject: [PATCH 1282/1515] [vk] Capture error message

---
 youtube_dl/extractor/vk.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 98bd3a141..9487fc9f5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -205,6 +205,12 @@ class VKIE(InfoExtractor):
 
         info_page = self._download_webpage(info_url, video_id)
 
+        error_message = self._html_search_regex(
+            r'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>',
+            info_page, 'error message', default=None)
+        if error_message:
+            raise ExtractorError(error_message, expected=True)
+
         if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
             raise ExtractorError(
                 'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',

From e58066e244bf9a04c2853e8829f7939bdefc9084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:25:06 +0600
Subject: [PATCH 1283/1515] [vk] Add age restricted video test for reference

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9487fc9f5..c30c5a8e5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -154,6 +154,11 @@ class VKIE(InfoExtractor):
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'only_matching': True,
         },
+        {
+            # age restricted video, requires vk account credentials
+            'url': 'https://vk.com/video205387401_164765225',
+            'only_matching': True,
+        },
         {
             # vk wrapper
             'url': 'http://www.biqle.ru/watch/847655_160197695',

From 3f5c6d0c1b955f9312b83fb0e1f3ae59d1230c82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:41:34 +0600
Subject: [PATCH 1284/1515] [francetv] Add support for embeds and clean up
 _VALID_URL

---
 youtube_dl/extractor/francetv.py | 44 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index b2c984bf2..827400172 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -160,11 +160,19 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
 class FranceTVIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetv'
     IE_DESC = 'France 2, 3, 4, 5 and Ô'
-    _VALID_URL = r'''(?x)https?://www\.france[2345o]\.fr/
-        (?:
-            emissions/.*?/(videos|emissions)/(?P<id>[^/?]+)
-        |   (emissions?|jt)/(?P<key>[^/?]+)
-        )'''
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?france[2345o]\.fr/
+                                (?:
+                                    emissions/[^/]+/(?:videos|diffusions)?|
+                                    videos
+                                )
+                            /|
+                            embed\.francetv\.fr/\?ue=
+                        )
+                        (?P<id>[^/?]+)
+                    '''
 
     _TESTS = [
         # france2
@@ -232,13 +240,33 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'timestamp': 1410822000,
             },
         },
+        {
+            # francetv embed
+            'url': 'http://embed.francetv.fr/?ue=8d7d3da1e3047c42ade5a5d7dfd3fc87',
+            'info_dict': {
+                'id': 'EV_30231',
+                'ext': 'flv',
+                'title': 'Alcaline, le concert avec Calogero',
+                'description': 'md5:',
+                'upload_date': '20150226',
+                'timestamp': 1424989860,
+            },
+        },
+        {
+            'url': 'http://www.france4.fr/emission/highlander/diffusion-du-17-07-2015-04h05',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.franceo.fr/videos/125377617',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        webpage = self._download_webpage(url, mobj.group('key') or mobj.group('id'))
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
         video_id, catalogue = self._html_search_regex(
-            r'href="http://videos\.francetv\.fr/video/([^@]+@[^"]+)"',
+            r'href="http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
             webpage, 'video ID').split('@')
         return self._extract_video(video_id, catalogue)
 

From 5705ee6ef8d49175bf4afac36af4b3b9a4e72596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:43:42 +0600
Subject: [PATCH 1285/1515] [francetv] Fix duration extraction

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 827400172..d4f98ca16 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -87,7 +87,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'title': info['titre'],
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
-            'duration': float_or_none(info.get('real_duration'), 1000) or parse_duration(info['duree']),
+            'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
         }
@@ -250,6 +250,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'description': 'md5:',
                 'upload_date': '20150226',
                 'timestamp': 1424989860,
+                'duration': 5400,
             },
         },
         {

From cbd55ade680c5a3ffe5f7b5d36dc01af2c6dc48d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:56:00 +0600
Subject: [PATCH 1286/1515] [extractor/generic] Add support for francetv embeds

---
 youtube_dl/extractor/generic.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a62287e50..dc24a8a8b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -407,6 +407,26 @@ class GenericIE(InfoExtractor):
                 'skip_download': 'Requires rtmpdump'
             }
         },
+        # francetv embed
+        {
+            'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
+            'info_dict': {
+                'id': 'EV_30231',
+                'ext': 'mp4',
+                'title': 'Alcaline, le concert avec Calogero',
+                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
+                'upload_date': '20150226',
+                'timestamp': 1424989860,
+                'duration': 5400,
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            },
+            'expected_warnings': [
+                'Forbidden'
+            ]
+        },
         # Condé Nast embed
         {
             'url': 'http://www.wired.com/2014/04/honda-asimo/',
@@ -1431,6 +1451,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
 
+        # Look for embedded francetv player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for embedded smotri.com player
         smotri_url = SmotriIE._extract_url(webpage)
         if smotri_url:

From 826a7da808448932d17c09a13a106d75ba0c10d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 00:27:23 +0600
Subject: [PATCH 1287/1515] [rtlnl] Avoid episodes completely (Closes #6275)

---
 youtube_dl/extractor/rtlnl.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index a4d3d73ff..e0c530d64 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -43,6 +43,21 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
+        'info_dict': {
+            'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',
+            'ext': 'mp4',
+            'title': 'RTL Nieuws - Meer beelden van overval juwelier',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
+            'timestamp': 1437233400,
+            'upload_date': '20150718',
+            'duration': 30.474,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # encrypted m3u8 streams, georestricted
         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
@@ -59,9 +74,11 @@ class RtlNlIE(InfoExtractor):
             uuid)
 
         material = info['material'][0]
-        progname = info['abstracts'][0]['name']
-        subtitle = material['title'] or info['episodes'][0]['name']
-        description = material.get('synopsis') or info['episodes'][0]['synopsis']
+        title = info['abstracts'][0]['name']
+        subtitle = material.get('title')
+        if subtitle:
+            title += ' - %s' % subtitle
+        description = material.get('synopsis')
 
         meta = info.get('meta', {})
 
@@ -107,7 +124,7 @@ class RtlNlIE(InfoExtractor):
 
         return {
             'id': uuid,
-            'title': '%s - %s' % (progname, subtitle),
+            'title': title,
             'formats': formats,
             'timestamp': material['original_date'],
             'description': description,

From 761ee0d827d17eefff0538a99c453e050b6e933e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 02:28:43 +0800
Subject: [PATCH 1288/1515] [iqiyi] Remove unused imports

---
 youtube_dl/extractor/iqiyi.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ac5416953..7caaa1183 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,19 +3,13 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
-import os.path
 import random
-import re
 import time
 import uuid
-import zlib
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import (
-    ExtractorError,
-    url_basename,
-)
+from ..utils import ExtractorError
 
 
 class IqiyiIE(InfoExtractor):

From 3af1fac7b0f43778e44b3b86e0c74bf25fb6f489 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 09:51:59 +0100
Subject: [PATCH 1289/1515] [dcn] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dcn.py      | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/dcn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..cc0da81d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -110,6 +110,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
+from .dcn import DcnIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
new file mode 100644
index 000000000..5263def4c
--- /dev/null
+++ b/youtube_dl/extractor/dcn.py
@@ -0,0 +1,46 @@
+from .common import InfoExtractor
+
+class DcnIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
+    _TEST = {
+        'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
+        'info_dict':
+        {
+            'id': '17375',
+            'ext': 'm3u8',
+            'title': 'رحلة العمر : الحلقة 1',
+            'description': '"في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة1"',
+            'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
+            'duration': '2041'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id='+video_id,
+            video_id
+        )
+        title = json_data['title_ar'];
+        thumbnail = 'http://admin.mangomolo.com/analytics/'+json_data['img'];
+        duration = json_data['duration'];
+        description = json_data['description_ar'];
+        webpage = self._download_webpage(
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id='+json_data['id']+'&user_id='+json_data['user_id']+'&countries=Q0M=&w=100%&h=100%&filter=DENY&signature='+json_data['signature'],
+            video_id
+        )
+        m3u8_url = self._html_search_regex(
+            r'file: "(?P<m3u8_url>.*?)"',
+            webpage,
+            'm3u8_url',
+            group='m3u8_url'
+        )
+        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'description': description,
+            'formats': formats,
+        }

From 9d681c2bb3b75a666b76d8e346ffab66b65f9132 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 10:00:24 +0100
Subject: [PATCH 1290/1515] remove unnecessary group name

---
 youtube_dl/extractor/dcn.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 5263def4c..f76ebda9e 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -30,10 +30,9 @@ class DcnIE(InfoExtractor):
             video_id
         )
         m3u8_url = self._html_search_regex(
-            r'file: "(?P<m3u8_url>.*?)"',
+            r'file:\s*"([^"]+)',
             webpage,
-            'm3u8_url',
-            group='m3u8_url'
+            'm3u8_url'
         )
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
         return {

From 36068ae019138710a3023334db2241ec815e2fe6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 03:54:59 +0800
Subject: [PATCH 1291/1515] [iqiyi] Comment out some MD5 sums

The value is different on Travis CI server
---
 youtube_dl/extractor/iqiyi.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 7caaa1183..d93b0867d 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -33,56 +33,48 @@ class IqiyiIE(InfoExtractor):
             'title': '名侦探柯南第752集',
         },
         'playlist': [{
-            'md5': '7e49376fecaffa115d951634917fe105',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '41b75ba13bb7ac0e411131f92bc4f6ca',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '0cee1dd0a3d46a83e71e2badeae2aab0',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '4f8ad72373b0c491b582e7c196b0b1f9',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': 'd89ad028bcfad282918e8098e811711d',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '9cb1e5c95da25dff0660c32ae50903b7',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '155116e0ff1867bbc9b98df294faabc9',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '53f5db77622ae14fa493ed2a278a082b',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
                 'ext': 'f4v',

From e89d7e30294d3c5a36e6af5dd730ed543934d40e Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 03:34:22 +0200
Subject: [PATCH 1292/1515] [tagesschau] add support for more video types
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I found that currently only tagesschau videos are played. There are some
more shows hosted on tagesschau.de (see [0] for example) which are
easily playable by adjusting the regex. So this patch adds support for:
* tagesthemen
* tagesschau vor 20 Jahren
* tagesschau (mit Gebärdensprache)
* nachtmagazin

Note that some videos don't provide a description, so in order for the
tests to succeed, an ExtractorError needs to get caught.

[0]: http://www.tagesschau.de/multimedia/video/videoarchiv2~_date-20150714.html
---
 youtube_dl/extractor/tagesschau.py | 55 +++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index bfe07b024..2a2aafca0 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize
+from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -30,6 +30,46 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
+        'md5': '90757268b49ef56deae90c7b48928d58',
+        'info_dict': {
+            'id': '3771',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
+        'md5': '6e3ebdc75e8d67da966a8d06721eda71',
+        'info_dict': {
+            'id': '3827',
+            'ext': 'mp4',
+            'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66',
+            'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
+        'md5': '8a8875a568f0a5ae5ceef93c501a225f',
+        'info_dict': {
+            'id': '3475',
+            'ext': 'mp4',
+            'description': 'md5:ed149f5649cda3dac86813a9d777e131',
+            'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
+        'md5': 'be4d6f0421f2acd8abe25ea29f6f015b',
+        'info_dict': {
+            'id': '959',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {
@@ -102,9 +142,14 @@ class TagesschauIE(InfoExtractor):
             thumbnail_fn = self._search_regex(
                 r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
                 webpage, 'thumbnail', fatal=False)
-            description = self._html_search_regex(
-                r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', fatal=False)
+            # there are some videos without description
+            description = ""
+            try:
+                description = self._html_search_regex(
+                    r'(?s)<p class="teasertext">(.*?)</p>',
+                    webpage, 'description', fatal=False)
+            except ExtractorError:
+                pass
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From c51bc70e0feec42664c57228cc4b395a3a629420 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 03:52:54 +0200
Subject: [PATCH 1293/1515] [tagesschau] fix test which links to nonexisting
 video

---
 youtube_dl/extractor/tagesschau.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index bfe07b024..cf1b37a75 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -11,13 +11,13 @@ class TagesschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
-        'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
-        'md5': 'bcdeac2194fb296d599ce7929dfa4009',
+        'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
+        'md5': '917a228bc7df7850783bc47979673a09',
         'info_dict': {
-            'id': '1399128',
+            'id': '102143',
             'ext': 'mp4',
-            'title': 'Harald Range, Generalbundesanwalt, zu den Ermittlungen',
-            'description': 'md5:69da3c61275b426426d711bde96463ab',
+            'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
+            'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
             'thumbnail': 're:^http:.*\.jpg$',
         },
     }, {

From 3c6ae8b59ed2517891ff19507e4cef2322ef9eb4 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 04:27:58 +0200
Subject: [PATCH 1294/1515] [tagesschau] add support for Bericht aus Berlin
 videos

---
 youtube_dl/extractor/tagesschau.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 2a2aafca0..682f8df8f 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -70,6 +70,16 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
+        'md5': '42e3757018d9908581481a80cc1806da',
+        'info_dict': {
+            'id': '3299',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {

From 726adc43ec184b6d7e05cdbc197aa02c53b32347 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 05:09:29 +0200
Subject: [PATCH 1295/1515] [tagesschau] set description=None for empty
 descriptions

---
 youtube_dl/extractor/tagesschau.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 682f8df8f..4a755c657 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -36,7 +36,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '3771',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -66,7 +66,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '959',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -76,7 +76,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '3299',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -154,12 +154,9 @@ class TagesschauIE(InfoExtractor):
                 webpage, 'thumbnail', fatal=False)
             # there are some videos without description
             description = ""
-            try:
-                description = self._html_search_regex(
-                    r'(?s)<p class="teasertext">(.*?)</p>',
-                    webpage, 'description', fatal=False)
-            except ExtractorError:
-                pass
+            description = self._html_search_regex(
+                r'(?s)<p class="teasertext">(.*?)</p>',
+                webpage, 'description', fatal=False, default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From 4951c9f8214153d84033a77a4ef798ae29e1466a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 11:16:29 +0800
Subject: [PATCH 1296/1515] Credit @sceext2 for fixing iQiyi extractor (#6266)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 531ec5767..4fd65f46f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -132,3 +132,4 @@ George Brighton
 Remita Amine
 Aurélio A. Heckert
 Bernhard Minks
+sceext

From c2d1be8981c0f6b91635c32a3163e01b91d5db54 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 11:20:05 +0800
Subject: [PATCH 1297/1515] [iqiyi] Add skip_download to reduce network traffic

MD5 checksums are commented out in
36068ae019138710a3023334db2241ec815e2fe6, and actual downloads are not
necessary anymore.
---
 youtube_dl/extractor/iqiyi.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d93b0867d..afb7f4e61 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -81,6 +81,9 @@ class IqiyiIE(InfoExtractor):
                 'title': '名侦探柯南第752集',
             },
         }],
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _FORMATS_MAP = [

From 0215103e92c7a40aa0cea319e72dcf49f02053fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:13:27 +0200
Subject: [PATCH 1298/1515] [francetv] Remove unused import

---
 youtube_dl/extractor/francetv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d4f98ca16..477e22793 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -14,7 +14,6 @@ from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
-    float_or_none,
     parse_duration,
     determine_ext,
 )

From 8a7a2089050957c942bde0b9ab70963b2dea8f35 Mon Sep 17 00:00:00 2001
From: slangangular <slangangular@infrablue.org>
Date: Sun, 12 Jul 2015 10:30:51 +0200
Subject: [PATCH 1299/1515] [sportschau] Add support for sportschau.de

Closes #6199.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/sportschau.py | 43 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..d70de7690 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -552,6 +552,7 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
+from .sportschau import SportschauIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
new file mode 100644
index 000000000..ff30e1f0e
--- /dev/null
+++ b/youtube_dl/extractor/sportschau.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SportschauIE(InfoExtractor):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
+        'md5': 'a6ef460ab9f4089b079832e06d554cec',
+        'info_dict': {
+            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
+            'ext': 'mp4',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        ext = '-mc_defaultQuality-h.json'
+        json_url = url[:-5] + ext
+
+        json = self._download_json(json_url, video_id)
+        thumb_url = json['_previewImage']
+
+        m3u8_url = json['_mediaArray'][1]['_mediaStreamArray'][0]['_stream'][0]
+        m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        desc = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': m3u8_formats,
+            'description': desc,
+            'thumbnail': thumb_url,
+        }

From 8b61bfd6389b62f054cdf9dcb3436395c82a8e28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:21:18 +0200
Subject: [PATCH 1300/1515] [sportschau] skip download in test

---
 youtube_dl/extractor/sportschau.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
index ff30e1f0e..28797266f 100644
--- a/youtube_dl/extractor/sportschau.py
+++ b/youtube_dl/extractor/sportschau.py
@@ -9,14 +9,17 @@ class SportschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
     _TEST = {
         'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'md5': 'a6ef460ab9f4089b079832e06d554cec',
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
             'ext': 'mp4',
             'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 32470bf619d31605dc9c51ad107839a097f829f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:24:19 +0200
Subject: [PATCH 1301/1515] [sportschau] Improve title extraction

The html '<title>' ends with '- sportschau.de', which shouldn't be part of the title.
---
 youtube_dl/extractor/sportschau.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
index 28797266f..bf9b075db 100644
--- a/youtube_dl/extractor/sportschau.py
+++ b/youtube_dl/extractor/sportschau.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import get_element_by_attribute
 
 
 class SportschauIE(InfoExtractor):
@@ -12,7 +13,7 @@ class SportschauIE(InfoExtractor):
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
             'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
         },
@@ -34,7 +35,7 @@ class SportschauIE(InfoExtractor):
         m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        title = get_element_by_attribute('class', 'headline', webpage)
         desc = self._html_search_meta('description', webpage)
 
         return {

From 1dc31c2786b34f833acc5fc646afcf992f71444d Mon Sep 17 00:00:00 2001
From: Zach Bruggeman <mail@bruggie.com>
Date: Fri, 10 Jul 2015 19:27:48 -0700
Subject: [PATCH 1302/1515] [appleconnect] Add new extractor (fixes #6189)

Closes #6190.
---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/appleconnect.py | 50 ++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/appleconnect.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d70de7690..5033d67ed 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,6 +19,7 @@ from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
+from .appleconnect import AppleConnectIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
 from .ard import ARDIE, ARDMediathekIE
diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dl/extractor/appleconnect.py
new file mode 100644
index 000000000..ea7a70393
--- /dev/null
+++ b/youtube_dl/extractor/appleconnect.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    str_to_int,
+    ExtractorError
+)
+
+
+class AppleConnectIE(InfoExtractor):
+    _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
+    _TEST = {
+        'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
+        'md5': '10d0f2799111df4cb1c924520ca78f98',
+        'info_dict': {
+            'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
+            'ext': 'm4v',
+            'title': 'Energy',
+            'uploader': 'Drake',
+            'thumbnail': 'http://is5.mzstatic.com/image/thumb/Video5/v4/78/61/c5/7861c5fa-ad6d-294b-1464-cf7605b911d6/source/1920x1080sr.jpg',
+            'upload_date': '20150710',
+            'timestamp': 1436545535,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        try:
+            video_json = self._html_search_regex(
+                r'class="auc-video-data">(\{.*?\})', webpage, 'json')
+        except ExtractorError:
+            raise ExtractorError('This post doesn\'t contain a video', expected=True)
+
+        video_data = self._parse_json(video_json, video_id)
+        timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
+        like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count'))
+
+        return {
+            'id': video_id,
+            'url': video_data['sslSrc'],
+            'title': video_data['title'],
+            'description': video_data['description'],
+            'uploader': video_data['artistName'],
+            'thumbnail': video_data['artworkUrl'],
+            'timestamp': timestamp,
+            'like_count': like_count,
+        }

From ecdbe09e1017fdd771720f9954f8427b9084bbf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:45:49 +0600
Subject: [PATCH 1303/1515] [francetv] Fix f4m extraction

---
 youtube_dl/extractor/francetv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 477e22793..624895ff9 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -58,9 +58,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     # See https://github.com/rg3/youtube-dl/issues/3963
                     # m3u8 urls work fine
                     continue
-                video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
-                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url_parsed.path,
+                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))

From 789a12aaafe7036a155435d9c93d062309f547b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:50:25 +0600
Subject: [PATCH 1304/1515] [francetv] Restore support for jt videos

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 624895ff9..11db6d921 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -164,7 +164,8 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                             (?:www\.)?france[2345o]\.fr/
                                 (?:
                                     emissions/[^/]+/(?:videos|diffusions)?|
-                                    videos
+                                    videos|
+                                    jt
                                 )
                             /|
                             embed\.francetv\.fr/\?ue=

From 632cbb8efa32ef704219a769972fdeae04577ab8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:51:06 +0600
Subject: [PATCH 1305/1515] [francetv] Fix f4m extraction completely

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 11db6d921..746ab9d84 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -62,7 +62,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
-                    formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))
+                    formats.extend(self._extract_f4m_formats(
+                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, 1, format_id))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
             elif video_url.startswith('rtmp'):

From 3bc9fb58892d0ca3dfc044b6e0ef65426ecd8c1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:53:09 +0600
Subject: [PATCH 1306/1515] [francetv] Update jt test

---
 youtube_dl/extractor/francetv.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 746ab9d84..f5aeab110 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -229,15 +229,16 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         },
         # franceo
         {
-            'url': 'http://www.franceo.fr/jt/info-afrique/04-12-2013',
-            'md5': '52f0bfe202848b15915a2f39aaa8981b',
+            'url': 'http://www.franceo.fr/jt/info-soir/18-07-2015',
+            'md5': '47d5816d3b24351cdce512ad7ab31da8',
             'info_dict': {
-                'id': '108634970',
+                'id': '125377621',
                 'ext': 'flv',
-                'title': 'Infô Afrique',
-                'description': 'md5:ebf346da789428841bee0fd2a935ea55',
-                'upload_date': '20140915',
-                'timestamp': 1410822000,
+                'title': 'Infô soir',
+                'description': 'md5:01b8c6915a3d93d8bbbd692651714309',
+                'upload_date': '20150718',
+                'timestamp': 1437241200,
+                'duration': 414,
             },
         },
         {

From ac4b8df5e47a113da5bb66fc975f938bc176b58c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:53:54 +0600
Subject: [PATCH 1307/1515] [francetv] Fix embed test

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index f5aeab110..08b124805 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -248,7 +248,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'id': 'EV_30231',
                 'ext': 'flv',
                 'title': 'Alcaline, le concert avec Calogero',
-                'description': 'md5:',
+                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
                 'upload_date': '20150226',
                 'timestamp': 1424989860,
                 'duration': 5400,

From 1d18e26ecad636cb88ee9710807e9583eec98cd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:54:12 +0600
Subject: [PATCH 1308/1515] [francetv] Remove unused import

---
 youtube_dl/extractor/francetv.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 08b124805..706ed9c99 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -6,10 +6,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     clean_html,
     ExtractorError,

From 7a4a945f138078b95f02e8cec1e38123d66d7f09 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 11:31:38 -0500
Subject: [PATCH 1309/1515] fix/support news.vice.com

---
 youtube_dl/extractor/vice.py | 40 ++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 04e2b0ba7..dfb621468 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,20 +7,34 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(.+?\.)?vice\.com/.*?/(?P<name>.+)'
 
-    _TEST = {
-        'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
-        'info_dict': {
-            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
-            'ext': 'mp4',
-            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-        },
-        'params': {
-            # Requires ffmpeg (m3u8 manifest)
-            'skip_download': True,
-        },
-    }
+    _TESTS = [
+        {
+            'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
+            'info_dict': {
+                'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+                'ext': 'mp4',
+                'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+            },
+            'params': {
+                # Requires ffmpeg (m3u8 manifest)
+                'skip_download': True,
+            },
+        }, {
+            'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+            'info_dict': {
+                'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
+                'ext': 'mp4',
+                'title': 'VICE News - Inside the Monkey Lab',
+                'description': 'md5:1f660d467d3515f29d11e5ef742a4b82',
+            },
+            'params': {
+                # Requires ffmpeg (m3u8 manifest)
+                'skip_download': True,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From a5dd9a0c5dcf6aace1b7af87acb982480d606753 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 11:47:58 -0500
Subject: [PATCH 1310/1515] pbs: fix vague 'Full Episode' titles; prepend name
 of show

Many videos are given the title 'Full Episode' or 'Episode x', etc;
in the info json 'title' is the episode title (for oneshots this is
'Full Episode') while the program name is 'program':{'title'. This
changes the title from '(episode title)' to
 '(program name) - (episode title)'

The following urls demonstrate useless titles

    http://video.pbs.org/video/2365367186/
    http://video.pbs.org/video/2365519307/
    http://video.pbs.org/video/2365527039/
    http://video.pbs.org/video/2365530605/

Before change

    Full Episode
    Episode 5 | Preview
    Season 3 | Episode 4 Preview
    July 17, 2015

After change

    To Catch A Comet - Full Episode
    The Crimson Field - Episode 5 | Preview
    Last Tango in Halifax - Season 3 | Episode 4 Preview
    Charlie Rose The Week - July 17, 2015
---
 youtube_dl/extractor/pbs.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index fec5d65ad..cccff270b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -32,7 +32,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365006249',
                 'ext': 'mp4',
-                'title': 'A More Perfect Union',
+                'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
                 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
                 'duration': 3190,
             },
@@ -46,7 +46,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365297690',
                 'ext': 'mp4',
-                'title': 'Losing Iraq',
+                'title': 'FRONTLINE - Losing Iraq',
                 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
                 'duration': 5050,
             },
@@ -60,7 +60,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2201174722',
                 'ext': 'mp4',
-                'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
+                'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
                 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
                 'duration': 801,
             },
@@ -72,7 +72,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297708',
                 'ext': 'mp4',
                 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
-                'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+                'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -88,7 +88,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'killer-typhoon',
                 'ext': 'mp4',
                 'description': 'md5:c741d14e979fc53228c575894094f157',
-                'title': 'Killer Typhoon',
+                'title': 'NOVA - Killer Typhoon',
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
@@ -110,7 +110,7 @@ class PBSIE(InfoExtractor):
                 'id': '2280706814',
                 'display_id': 'player',
                 'ext': 'mp4',
-                'title': 'Death and the Civil War',
+                'title': 'American Experience - Death and the Civil War',
                 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
                 'duration': 6705,
                 'thumbnail': 're:^https?://.*\.jpg$',
@@ -118,6 +118,21 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            'url': 'http://video.pbs.org/video/2365367186/',
+            'info_dict': {
+                'id': '2365367186',
+                'display_id': '2365367186',
+                'ext': 'mp4',
+                'title': 'To Catch A Comet - Full Episode',
+                'description': 'On November 12, 2014, billions of kilometers from Earth, spacecraft orbiter Rosetta and lander Philae did what no other had dared to attempt \u2014 land on the volatile surface of a comet as it zooms around the sun at 67,000 km/hr. The European Space Agency hopes this mission can help peer into our past and unlock secrets of our origins.',
+                'duration': 3342,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
 
@@ -232,6 +247,12 @@ class PBSIE(InfoExtractor):
                 'url': closed_captions_url,
             }]
 
+        # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
+        # 'Full Episode', 'Episode 5', etc. prepend program->title
+        alt_title = info.get('program', {}).get('title', '')
+        if alt_title != '':
+            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
+
         return {
             'id': video_id,
             'display_id': display_id,

From e37c92ec6d137cd2d09c5f13e9a261c4fae552dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 23:59:50 +0600
Subject: [PATCH 1311/1515] [ard] Extract all formats

---
 youtube_dl/extractor/ard.py | 125 ++++++++++++++++++++++++++----------
 1 file changed, 91 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6a35ea463..55f940d57 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -35,6 +35,87 @@ class ARDMediathekIE(InfoExtractor):
         'skip': 'Blocked outside of Germany',
     }]
 
+    def _extract_media_info(self, media_info_url, webpage, video_id):
+        media_info = self._download_json(
+            media_info_url, video_id, 'Downloading media JSON')
+
+        formats = self._extract_formats(media_info, video_id)
+
+        if not formats:
+            if '"fsk"' in webpage:
+                raise ExtractorError(
+                    'This video is only available after 20:00', expected=True)
+            elif media_info.get('_geoblocked'):
+                raise ExtractorError('This video is not available due to geo restriction', expected=True)
+
+        self._sort_formats(formats)
+
+        duration = int_or_none(media_info.get('_duration'))
+        thumbnail = media_info.get('_previewImage')
+
+        subtitles = {}
+        subtitle_url = media_info.get('_subtitleUrl')
+        if subtitle_url:
+            subtitles['de'] = [{
+                'ext': 'srt',
+                'url': subtitle_url,
+            }]
+
+        return {
+            'id': video_id,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _extract_formats(self, media_info, video_id):
+        type_ = media_info.get('_type')
+        media_array = media_info.get('_mediaArray', [])
+        formats = []
+        for num, media in enumerate(media_array):
+            for stream in media.get('_mediaStreamArray', []):
+                stream_urls = stream.get('_stream')
+                if not stream_urls:
+                    continue
+                if not isinstance(stream_urls, list):
+                    stream_urls = [stream_urls]
+                quality = stream.get('_quality')
+                server = stream.get('_server')
+                for stream_url in stream_urls:
+                    ext = determine_ext(stream_url)
+                    if ext == 'f4m':
+                        formats.extend(self._extract_f4m_formats(
+                            stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
+                            video_id, preference=-1, f4m_id='hds'))
+                    elif ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls'))
+                    else:
+                        if server and server.startswith('rtmp'):
+                            f = {
+                                'url': server,
+                                'play_path': stream_url,
+                                'format_id': 'a%s-rtmp-%s' % (num, quality),
+                            }
+                        elif stream_url.startswith('http'):
+                            f = {
+                                'url': stream_url,
+                                'format_id': 'a%s-%s-%s' % (num, ext, quality)
+                            }
+                        else:
+                            continue
+                        m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', stream_url)
+                        if m:
+                            f.update({
+                                'width': int(m.group('width')),
+                                'height': int(m.group('height')),
+                            })
+                        if type_ == 'audio':
+                            f['vcodec'] = 'none'
+                        formats.append(f)
+        return formats
+
     def _real_extract(self, url):
         # determine video id from url
         m = re.match(self._VALID_URL, url)
@@ -92,46 +173,22 @@ class ARDMediathekIE(InfoExtractor):
                     'format_id': fid,
                     'url': furl,
                 })
+            self._sort_formats(formats)
+            info = {
+                'formats': formats,
+            }
         else:  # request JSON file
-            media_info = self._download_json(
-                'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
-            # The second element of the _mediaArray contains the standard http urls
-            streams = media_info['_mediaArray'][1]['_mediaStreamArray']
-            if not streams:
-                if '"fsk"' in webpage:
-                    raise ExtractorError('This video is only available after 20:00')
+            info = self._extract_media_info(
+                'http://www.ardmediathek.de/play/media/%s' % video_id, webpage, video_id)
 
-            formats = []
-            for s in streams:
-                if type(s['_stream']) == list:
-                    for index, url in enumerate(s['_stream'][::-1]):
-                        quality = s['_quality'] + index
-                        formats.append({
-                            'quality': quality,
-                            'url': url,
-                            'format_id': '%s-%s' % (determine_ext(url), quality)
-                        })
-                    continue
-
-                format = {
-                    'quality': s['_quality'],
-                    'url': s['_stream'],
-                }
-
-                format['format_id'] = '%s-%s' % (
-                    determine_ext(format['url']), format['quality'])
-
-                formats.append(format)
-
-        self._sort_formats(formats)
-
-        return {
+        info.update({
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': formats,
             'thumbnail': thumbnail,
-        }
+        })
+
+        return info
 
 
 class ARDIE(InfoExtractor):

From 3bf8c316a663741ab806f48cf9121209aba88b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:01:22 +0600
Subject: [PATCH 1312/1515] [sportschau] Reimplement in terms of ard extractor

---
 youtube_dl/extractor/__init__.py   |  7 +++--
 youtube_dl/extractor/ard.py        | 39 +++++++++++++++++++++++++
 youtube_dl/extractor/sportschau.py | 47 ------------------------------
 3 files changed, 44 insertions(+), 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5033d67ed..50da08830 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -22,7 +22,11 @@ from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
-from .ard import ARDIE, ARDMediathekIE
+from .ard import (
+    ARDIE,
+    ARDMediathekIE,
+    SportschauIE,
+)
 from .arte import (
     ArteTvIE,
     ArteTVPlus7IE,
@@ -553,7 +557,6 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
-from .sportschau import SportschauIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 55f940d57..2c368d833 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -8,6 +8,7 @@ from .generic import GenericIE
 from ..utils import (
     determine_ext,
     ExtractorError,
+    get_element_by_attribute,
     qualities,
     int_or_none,
     parse_duration,
@@ -246,3 +247,41 @@ class ARDIE(InfoExtractor):
             'upload_date': upload_date,
             'thumbnail': thumbnail,
         }
+
+
+class SportschauIE(ARDMediathekIE):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
+        'info_dict': {
+            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
+            'ext': 'mp4',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        base_url = mobj.group('baseurl')
+
+        webpage = self._download_webpage(url, video_id)
+        title = get_element_by_attribute('class', 'headline', webpage)
+        description = self._html_search_meta('description', webpage, 'description')
+
+        info = self._extract_media_info(
+            base_url + '-mc_defaultQuality-h.json', webpage, video_id)
+
+        info.update({
+            'title': title,
+            'description': description,
+        })
+
+        return info
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
deleted file mode 100644
index bf9b075db..000000000
--- a/youtube_dl/extractor/sportschau.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import get_element_by_attribute
-
-
-class SportschauIE(InfoExtractor):
-    IE_NAME = 'Sportschau'
-    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
-    _TEST = {
-        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'info_dict': {
-            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
-            'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        ext = '-mc_defaultQuality-h.json'
-        json_url = url[:-5] + ext
-
-        json = self._download_json(json_url, video_id)
-        thumb_url = json['_previewImage']
-
-        m3u8_url = json['_mediaArray'][1]['_mediaStreamArray'][0]['_stream'][0]
-        m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
-
-        webpage = self._download_webpage(url, video_id)
-        title = get_element_by_attribute('class', 'headline', webpage)
-        desc = self._html_search_meta('description', webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': m3u8_formats,
-            'description': desc,
-            'thumbnail': thumb_url,
-        }

From 86b4e98ac6f57c3703cd2082a69351ece797111c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:05:10 +0600
Subject: [PATCH 1313/1515] [ard:mediathek] Add audio test

---
 youtube_dl/extractor/ard.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 2c368d833..fdb94af6b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -34,6 +34,17 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
         },
         'skip': 'Blocked outside of Germany',
+    }, {
+        # audio
+        'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
+        'md5': '219d94d8980b4f538c7fcb0865eb7f2c',
+        'info_dict': {
+            'id': '28488308',
+            'ext': 'mp3',
+            'title': 'Tod eines Fußballers',
+            'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
+            'duration': 3240,
+        },
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):
@@ -252,7 +263,7 @@ class ARDIE(InfoExtractor):
 class SportschauIE(ARDMediathekIE):
     IE_NAME = 'Sportschau'
     _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
@@ -265,7 +276,7 @@ class SportschauIE(ARDMediathekIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 769efa16afc1c9a2871543f69081f1bced361dc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:08:21 +0600
Subject: [PATCH 1314/1515] [ard:mediathek] Remove dead test

---
 youtube_dl/extractor/ard.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index fdb94af6b..c78475748 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -23,17 +23,18 @@ class ARDMediathekIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
 
     _TESTS = [{
-        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
+        'url': 'http://www.ardmediathek.de/tv/Dokumentation-und-Reportage/Ich-liebe-das-Leben-trotzdem/rbb-Fernsehen/Video?documentId=29582122&bcastId=3822114',
         'info_dict': {
-            'id': '22490580',
+            'id': '29582122',
             'ext': 'mp4',
-            'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
-            'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
+            'title': 'Ich liebe das Leben trotzdem',
+            'description': 'md5:45e4c225c72b27993314b31a84a5261c',
+            'duration': 4557,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-        'skip': 'Blocked outside of Germany',
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
@@ -45,6 +46,9 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
             'duration': 3240,
         },
+    }, {
+        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
+        'only_matching': True,
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):

From d719c6a5abf746c31f301ed1a1d8934715eaeb05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:13:39 +0600
Subject: [PATCH 1315/1515] [ard:mediathek] Add test for direct mp4

---
 youtube_dl/extractor/ard.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index c78475748..6f465789b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -35,6 +35,16 @@ class ARDMediathekIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.ardmediathek.de/tv/Tatort/Tatort-Scheinwelten-H%C3%B6rfassung-Video/Das-Erste/Video?documentId=29522730&bcastId=602916',
+        'md5': 'f4d98b10759ac06c0072bbcd1f0b9e3e',
+        'info_dict': {
+            'id': '29522730',
+            'ext': 'mp4',
+            'title': 'Tatort: Scheinwelten - Hörfassung (Video tgl. ab 20 Uhr)',
+            'description': 'md5:196392e79876d0ac94c94e8cdb2875f1',
+            'duration': 5252,
+        },
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',

From 772acaf31f4614d390a445fe4c0d4e533c165774 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:39:55 +0600
Subject: [PATCH 1316/1515] [vice] Do not capture unused groups in _VALID_URL

---
 youtube_dl/extractor/vice.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index dfb621468..e3ad4ef54 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(.+?\.)?vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/.*?/(?P<name>.+)'
 
     _TESTS = [
         {

From 65c2b21df15053581cac21690605d1f7ac834083 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:40:43 +0600
Subject: [PATCH 1317/1515] [vice] Make test only matching

---
 youtube_dl/extractor/vice.py | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index e3ad4ef54..ae1faca1c 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -23,16 +23,7 @@ class ViceIE(InfoExtractor):
             },
         }, {
             'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-            'info_dict': {
-                'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
-                'ext': 'mp4',
-                'title': 'VICE News - Inside the Monkey Lab',
-                'description': 'md5:1f660d467d3515f29d11e5ef742a4b82',
-            },
-            'params': {
-                # Requires ffmpeg (m3u8 manifest)
-                'skip_download': True,
-            },
+            'only_matching': True,
         }
     ]
 

From 18ae46ad4bcf4d427984e85a9feaf647c3f80737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:42:25 +0600
Subject: [PATCH 1318/1515] [vice] Modernize

---
 youtube_dl/extractor/vice.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index ae1faca1c..01af7a995 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -1,5 +1,4 @@
 from __future__ import unicode_literals
-import re
 
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
@@ -7,7 +6,7 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)+(?P<id>.+)'
 
     _TESTS = [
         {
@@ -28,9 +27,8 @@ class ViceIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
         try:
             embed_code = self._search_regex(
                 r'embedCode=([^&\'"]+)', webpage,

From 0eacd2aaae7d4150d9cf4e1dd8ffc2ebc7ab030b Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 13:59:12 -0500
Subject: [PATCH 1319/1515] less clunky if statement

---
 youtube_dl/extractor/pbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cccff270b..ccbe2a9f3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -249,8 +249,8 @@ class PBSIE(InfoExtractor):
 
         # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
         # 'Full Episode', 'Episode 5', etc. prepend program->title
-        alt_title = info.get('program', {}).get('title', '')
-        if alt_title != '':
+        alt_title = info.get('program', {}).get('title')
+        if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
 
         return {

From 308c505c3d66b2a11df8c1a5cd7e0fdaa694b585 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 01:03:43 +0600
Subject: [PATCH 1320/1515] [francetv] Improve _VALID_URL

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 706ed9c99..75723c00d 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -161,7 +161,8 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                         (?:
                             (?:www\.)?france[2345o]\.fr/
                                 (?:
-                                    emissions/[^/]+/(?:videos|diffusions)?|
+                                    emissions/[^/]+/(?:videos|diffusions)|
+                                    emission/[^/]+|
                                     videos|
                                     jt
                                 )

From faa1f83ab481a1f456db13d3ead7f0f22908aab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 02:16:07 +0600
Subject: [PATCH 1321/1515] [twitch:stream] Improve _VALID_URL (Closes #6294)

---
 youtube_dl/extractor/twitch.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 948c8ce39..73ce335b7 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -310,9 +310,9 @@ class TwitchBookmarksIE(TwitchPlaylistBaseIE):
 
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',
         'info_dict': {
             'id': '12772022048',
@@ -331,7 +331,10 @@ class TwitchStreamIE(TwitchBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)

From a1b85269a49404cab1f549d7c59352fe48dfbf19 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:49:53 +0800
Subject: [PATCH 1322/1515] [extractor/generic] Support vid.me embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc24a8a8b..cd133a10c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1196,6 +1196,12 @@ class GenericIE(InfoExtractor):
         if vimeo_url is not None:
             return self.url_result(vimeo_url)
 
+        vid_me_embed_url = self._search_regex(
+            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
+            webpage, 'vid.me embed', default=None)
+        if vid_me_embed_url is not None:
+            return self.url_result(vid_me_embed_url, 'Vidme')
+
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
             (?:

From c909e5820e94ecfefd9a2eb41b1beaa684c530c4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:51:25 +0800
Subject: [PATCH 1323/1515] [tumblr] Delegate to GenericIE for non-tumblr
 videos

Fixes #6162
---
 youtube_dl/extractor/tumblr.py | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 9ead13a91..772f94f7d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,8 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .pornhub import PornHubIE
-from .vimeo import VimeoIE
 
 
 class TumblrIE(InfoExtractor):
@@ -60,25 +58,14 @@ class TumblrIE(InfoExtractor):
         blog = m_url.group('blog_name')
 
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
-        webpage = self._download_webpage(url, video_id)
-
-        vid_me_embed_url = self._search_regex(
-            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
-            webpage, 'vid.me embed', default=None)
-        if vid_me_embed_url is not None:
-            return self.url_result(vid_me_embed_url, 'Vidme')
-
-        pornhub_url = PornHubIE._extract_url(webpage)
-        if pornhub_url:
-            return self.url_result(pornhub_url, 'PornHub')
-
-        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
-        if vimeo_url:
-            return self.url_result(vimeo_url, 'Vimeo')
+        webpage, urlh = self._download_webpage_handle(url, video_id)
 
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
-            webpage, 'iframe url')
+            webpage, 'iframe url', default=None)
+        if iframe_url is None:
+            return self.url_result(urlh.geturl(), 'Generic')
+
         iframe = self._download_webpage(iframe_url, video_id)
         video_url = self._search_regex(r'<source src="([^"]+)"',
                                        iframe, 'video url')

From 3f302bca8cf29f618d1f1c2fb7bc4ea9538af805 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:54:47 +0800
Subject: [PATCH 1324/1515] [tumblr] Improve downloading notes

---
 youtube_dl/extractor/tumblr.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 772f94f7d..3d3b635e4 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -66,7 +66,8 @@ class TumblrIE(InfoExtractor):
         if iframe_url is None:
             return self.url_result(urlh.geturl(), 'Generic')
 
-        iframe = self._download_webpage(iframe_url, video_id)
+        iframe = self._download_webpage(iframe_url, video_id,
+                                        'Downloading iframe page')
         video_url = self._search_regex(r'<source src="([^"]+)"',
                                        iframe, 'video url')
 

From 17ee98e1a5a6897ed0d9c52d9257b45ed9c3f20a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 19:48:50 +0200
Subject: [PATCH 1325/1515] [youtube] Fix upload_date in test

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9a08924ef..f73032abd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -543,7 +543,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'CsmdDsKjzN8',
                 'ext': 'mp4',
-                'upload_date': '20150510',
+                'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
                 'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',

From 2fe1ff85829244f6498dde8c85ecdc1415f5ad7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:14:20 +0200
Subject: [PATCH 1326/1515] [youtube] Set 'is_live'

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f73032abd..3d8b31f98 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -915,6 +915,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Get video info
         embed_webpage = None
+        is_live = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -947,6 +948,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
+                if args.get('livestream') == '1' or args.get('live_playback') == 1:
+                    is_live = True
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1251,6 +1254,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'dislike_count': dislike_count,
             'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
+            'is_live': is_live,
         }
 
 

From 8250c32f49fed0986bd514d1435e00a0182e8314 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:25:53 +0200
Subject: [PATCH 1327/1515] [YoutubeDL] don't default to
 'bestvideo+bestaudio/best' for live videos

Doesn't work currently.
---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 00af78e06..702a6ad50 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1104,7 +1104,8 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    info_dict['extractor'] in ['youtube', 'ted']):
+                    info_dict['extractor'] in ['youtube', 'ted'] and
+                    not info_dict.get('is_live')):
                 merger = FFmpegMergerPP(self)
                 if merger.available and merger.can_merge():
                     req_format_list.append('bestvideo+bestaudio')

From 7c80519cbf7e3daa029239fbbd147652817877f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 21:10:28 +0200
Subject: [PATCH 1328/1515] [youtube] Extract start_time

From the 't=*' in the url.
Currently youtube-dl doesn't use the value, but it was requested for the mpv plugin.
---
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b9014fc23..9e8751877 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -183,6 +183,8 @@ class InfoExtractor(object):
                     ["Sports", "Berlin"]
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
+    start_time:     Time in seconds where the reproduction should start, as
+                    specified in the url.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d8b31f98..afbd34f4a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -19,6 +19,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
@@ -31,6 +32,7 @@ from ..utils import (
     get_element_by_id,
     int_or_none,
     orderedSet,
+    parse_duration,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -317,7 +319,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -329,6 +331,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'categories': ['Science & Technology'],
                 'like_count': int,
                 'dislike_count': int,
+                'start_time': 1,
             }
         },
         {
@@ -889,6 +892,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'http' if self._downloader.params.get('prefer_insecure', False)
             else 'https')
 
+        start_time = None
+        parsed_url = compat_urllib_parse_urlparse(url)
+        for component in [parsed_url.fragment, parsed_url.query]:
+            query = compat_parse_qs(component)
+            if 't' in query:
+                start_time = parse_duration(query['t'][0])
+                break
+
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
         if mobj:
@@ -1255,6 +1266,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
             'is_live': is_live,
+            'start_time': start_time,
         }
 
 

From f993afb26d20ac79891621494922887b3c8df02b Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Mon, 20 Jul 2015 23:10:29 +0200
Subject: [PATCH 1329/1515] README: fix a typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2cc89cdb..ac54d7b67 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
     --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8"
                                      if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
                                      download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
     --match-title REGEX              Download only matching titles (regex or caseless sub-string)

From 4eb59a6b1cd1c4ca5f7dd284f0c26a329bb8e973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jul 2015 09:11:23 +0600
Subject: [PATCH 1330/1515] [options] Fix a typo (#6307)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 85365d769..9016e3498 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -219,7 +219,7 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--playlist-items',
         dest='playlist_items', metavar='ITEM_SPEC', default=None,
-        help='Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
+        help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',

From ce9512b78bbd5f180f44670f6b78784a0be3b815 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 21 Jul 2015 17:20:54 +0200
Subject: [PATCH 1331/1515] release 2015.07.21

---
 docs/supportedsites.md | 6 ++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a84878026..73445137f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
+ - **AppleConnect**
  - **AppleDaily**: 臺灣蘋果日報
  - **AppleTrailers**
  - **archive.org**: archive.org videos
@@ -486,6 +487,7 @@
  - **SportBox**
  - **SportBoxEmbed**
  - **SportDeutschland**
+ - **Sportschau**
  - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
@@ -611,8 +613,8 @@
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
- - **vk.com**
- - **vk.com:user-videos**: vk.com:All of a user's videos
+ - **vk**: VK
+ - **vk:uservideos**: VK - User's Videos
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3ad7a2bc0..280afdd7f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.18'
+__version__ = '2015.07.21'

From 496ce6b3493757e806d857363955a2289410a4e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jul 2015 23:54:31 +0600
Subject: [PATCH 1332/1515] [snagfilms] Improve m3u8 extraction (Closes #6309)

---
 youtube_dl/extractor/snagfilms.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cf495f310..0e820d250 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -23,6 +23,15 @@ class SnagFilmsEmbedIE(InfoExtractor):
             'ext': 'mp4',
             'title': '#whilewewatch',
         }
+    }, {
+        # invalid labels, 360p is better that 480p
+        'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
+        'md5': '882fca19b9eb27ef865efeeaed376a48',
+        'info_dict': {
+            'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
+            'ext': 'mp4',
+            'title': 'Life in Limbo',
+        }
     }, {
         'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
         'only_matching': True,
@@ -52,14 +61,15 @@ class SnagFilmsEmbedIE(InfoExtractor):
             if not file_:
                 continue
             type_ = source.get('type')
-            format_id = source.get('label')
             ext = determine_ext(file_)
-            if any(_ == 'm3u8' for _ in (type_, ext)):
+            format_id = source.get('label') or ext
+            if all(_ == 'm3u8' for _ in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:
                 bitrate = int_or_none(self._search_regex(
-                    r'(\d+)kbps', file_, 'bitrate', default=None))
+                    [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
+                    file_, 'bitrate', default=None))
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]$', format_id, 'height', default=None))
                 formats.append({

From 28afa6e77a8fb95541ae9bead0ec0e57958cbfc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Jul 2015 20:50:02 +0200
Subject: [PATCH 1333/1515] [snagfilms] Don't use '_' as a variable that is
 used

---
 youtube_dl/extractor/snagfilms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 0e820d250..6977afb27 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -63,7 +63,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
             type_ = source.get('type')
             ext = determine_ext(file_)
             format_id = source.get('label') or ext
-            if all(_ == 'm3u8' for _ in (type_, ext)):
+            if all(v == 'm3u8' for v in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:

From b68a2613f856fbbf943abf8d6de6ffc6aff2e930 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 02:00:21 +0600
Subject: [PATCH 1334/1515] [viewster] Rewrite for new API (Closes #6317)

---
 youtube_dl/extractor/viewster.py | 218 ++++++++++++++++---------------
 1 file changed, 113 insertions(+), 105 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 1742e66f4..03c873608 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -1,129 +1,137 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class ViewsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
+    _VALID_URL = r'http://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
     _TESTS = [{
-        # movielink, paymethod=fre
-        'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
-        'playlist': [{
-            'md5': '8f9d94b282d80c42b378dffdbb11caf3',
-            'info_dict': {
-                'id': '1293-19341-000-movie',
-                'ext': 'flv',
-                'title': "'Hout' (Wood) - Movie",
-            },
-        }],
-        'info_dict': {
-            'id': '1293-19341-000',
-            'title': "'Hout' (Wood)",
-            'description': 'md5:925733185a9242ef96f436937683f33b',
-        }
-    }, {
-        # movielink, paymethod=adv
+        # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
-        'playlist': [{
-            'md5': '77a005453ca7396cbe3d35c9bea30aef',
-            'info_dict': {
-                'id': '1140-11855-000-movie',
-                'ext': 'flv',
-                'title': "THE LISTENING PROJECT - Movie",
-            },
-        }],
+        'md5': '14d3cfffe66d57b41ae2d9c873416f01',
         'info_dict': {
             'id': '1140-11855-000',
-            'title': "THE LISTENING PROJECT",
-            'description': 'md5:714421ae9957e112e672551094bf3b08',
-        }
+            'ext': 'flv',
+            'title': 'The listening Project',
+            'description': 'md5:bac720244afd1a8ea279864e67baa071',
+            'timestamp': 1214870400,
+            'upload_date': '20080701',
+            'duration': 4680,
+        },
     }, {
-        # direct links, no movielink
-        'url': 'http://www.viewster.com/movie/1198-56411-000/sinister/',
-        'playlist': [{
-            'md5': '0307b7eac6bfb21ab0577a71f6eebd8f',
-            'info_dict': {
-                'id': '1198-56411-000-trailer',
-                'ext': 'mp4',
-                'title': "Sinister - Trailer",
-            },
-        }, {
-            'md5': '80b9ee3ad69fb368f104cb5d9732ae95',
-            'info_dict': {
-                'id': '1198-56411-000-behind-scenes',
-                'ext': 'mp4',
-                'title': "Sinister - Behind Scenes",
-            },
-        }, {
-            'md5': '3b3ea897ecaa91fca57a8a94ac1b15c5',
-            'info_dict': {
-                'id': '1198-56411-000-scene-from-movie',
-                'ext': 'mp4',
-                'title': "Sinister - Scene from movie",
-            },
-        }],
+        # series episode, Type=Episode
+        'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
+        'md5': 'd5434c80fcfdb61651cc2199a88d6ba3',
         'info_dict': {
-            'id': '1198-56411-000',
-            'title': "Sinister",
-            'description': 'md5:014c40b0488848de9683566a42e33372',
-        }
+            'id': '1284-19427-001',
+            'ext': 'flv',
+            'title': 'The World and a Wall',
+            'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
+            'timestamp': 1428192000,
+            'upload_date': '20150405',
+            'duration': 1500,
+        },
+    }, {
+        # serie, Type=Serie
+        'url': 'http://www.viewster.com/serie/1303-19426-000/',
+        'info_dict': {
+            'id': '1303-19426-000',
+            'title': 'Is It Wrong to Try to Pick up Girls in a Dungeon?',
+            'description': 'md5:eeda9bef25b0d524b3a29a97804c2f11',
+        },
+        'playlist_count': 13,
+    }, {
+        # unfinished serie, no Type
+        'url': 'http://www.viewster.com/serie/1284-19427-000/baby-steps-season-2/',
+        'info_dict': {
+            'id': '1284-19427-000',
+            'title': 'Baby Steps—Season 2',
+            'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
+        },
+        'playlist_mincount': 16,
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
+    _AUTH_TOKEN = '/YqhSYsx8EaU9Bsta3ojlA=='
+
+    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
+        request = compat_urllib_request.Request(url)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+        request.add_header('Auth-token', self._AUTH_TOKEN)
+        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
-            'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
-        request.add_header('Accept', self._ACCEPT_HEADER)
+        info = self._download_json(
+            'https://public-api.viewster.com/search/%s' % video_id,
+            video_id, 'Downloading entry JSON')
 
-        movie = self._download_json(
-            request, video_id, 'Downloading movie metadata JSON')
+        entry_id = info.get('Id') or info['id']
 
-        title = movie.get('title') or movie['original_title']
-        description = movie.get('synopsis')
-        thumbnail = movie.get('large_artwork') or movie.get('artwork')
+        # unfinished serie has no Type
+        if info.get('Type') in ['Serie', None]:
+            episodes = self._download_json(
+                'https://public-api.viewster.com/series/%s/episodes' % entry_id,
+                video_id, 'Downloading series JSON')
+            entries = [
+                self.url_result(
+                    'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
+                for episode in episodes]
+            title = info.get('Title') or info['Synopsis']['Title']
+            description = info.get('Synopsis', {}).get('Detailed')
+            return self.playlist_result(entries, video_id, title, description)
 
-        entries = []
-        for clip in movie['play_list']:
-            entry = None
-
-            # movielink api
-            link_request = clip.get('link_request')
-            if link_request:
-                request = compat_urllib_request.Request(
-                    'http://api.live.viewster.com/api/v1/movielink?movieid=%(movieid)s&action=%(action)s&paymethod=%(paymethod)s&price=%(price)s&currency=%(currency)s&language=%(language)s&subtitlelanguage=%(subtitlelanguage)s&ischromecast=%(ischromecast)s'
-                    % link_request)
-                request.add_header('Accept', self._ACCEPT_HEADER)
-
-                movie_link = self._download_json(
-                    request, video_id, 'Downloading movie link JSON', fatal=False)
-
-                if movie_link:
-                    formats = self._extract_f4m_formats(
-                        movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
-                    self._sort_formats(formats)
-                    entry = {
-                        'formats': formats,
-                    }
-
-            # direct link
-            clip_url = clip.get('clip_data', {}).get('url')
-            if clip_url:
-                entry = {
-                    'url': clip_url,
-                    'ext': 'mp4',
-                }
-
-            if entry:
-                entry.update({
-                    'id': '%s-%s' % (video_id, clip['canonical_title']),
-                    'title': '%s - %s' % (title, clip['title']),
+        formats = []
+        for media_type in ('application/f4m+xml', 'application/x-mpegURL'):
+            media = self._download_json(
+                'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
+                % (entry_id, compat_urllib_parse.quote(media_type)),
+                video_id, 'Downloading %s JSON' % media_type, fatal=False)
+            if not media:
+                continue
+            video_url = media.get('Uri')
+            if not video_url:
+                continue
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
+                video_url += '&' if '?' in video_url else '?'
+                video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls',
+                    fatal=False  # m3u8 sometimes fail
+                ))
+            else:
+                formats.append({
+                    'url': video_url,
                 })
-                entries.append(entry)
+        self._sort_formats(formats)
 
-        playlist = self.playlist_result(entries, video_id, title, description)
-        playlist['thumbnail'] = thumbnail
-        return playlist
+        synopsis = info.get('Synopsis', {})
+        # Prefer title outside synopsis since it's less messy
+        title = info.get('Title') or synopsis['Title'].strip()
+        description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
+        duration = int_or_none(info.get('Duration'))
+        timestamp = parse_iso8601(info.get('ReleaseDate'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }

From c84683c88bbcb3cfd8d27af54a418035f431371d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 02:08:25 +0600
Subject: [PATCH 1335/1515] [viewster] Strip titles

---
 youtube_dl/extractor/viewster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 03c873608..6ef36290b 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -88,7 +88,7 @@ class ViewsterIE(InfoExtractor):
                 self.url_result(
                     'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
                 for episode in episodes]
-            title = info.get('Title') or info['Synopsis']['Title']
+            title = (info.get('Title') or info['Synopsis']['Title']).strip()
             description = info.get('Synopsis', {}).get('Detailed')
             return self.playlist_result(entries, video_id, title, description)
 
@@ -122,7 +122,7 @@ class ViewsterIE(InfoExtractor):
 
         synopsis = info.get('Synopsis', {})
         # Prefer title outside synopsis since it's less messy
-        title = info.get('Title') or synopsis['Title'].strip()
+        title = (info.get('Title') or synopsis['Title']).strip()
         description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
         duration = int_or_none(info.get('Duration'))
         timestamp = parse_iso8601(info.get('ReleaseDate'))

From 70c857b7283270146864f92d159445e038f40781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 11:49:54 +0200
Subject: [PATCH 1336/1515] Credit Zach Bruggeman for the appleconnect
 extractor (#6190)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4fd65f46f..373e05c9f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -133,3 +133,4 @@ Remita Amine
 Aurélio A. Heckert
 Bernhard Minks
 sceext
+Zach Bruggeman

From 981b9cdc8c12d817eaf3ec6b030538c252efe48e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 22:09:01 +0800
Subject: [PATCH 1337/1515] [lecture2go] Improve some regular expressions

---
 youtube_dl/extractor/lecture2go.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 9cf28e31c..fd115ff54 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class Lecture2GoIE(InfoExtractor):
-    _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
         'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
@@ -21,9 +21,9 @@ class Lecture2GoIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<em class="title">(.*?)</em>', webpage, 'title')
+        title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
         video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
-        creator = self._html_search_regex(r'<div id="description">(.*)</div>', webpage, 'creator')
+        creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
 
         return {
             'id': video_id,

From 795704f0f1f963d3f61a7e20074ce41eeb3cdf95 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 22:39:46 +0800
Subject: [PATCH 1338/1515] [lecture2go] Support more formats

---
 youtube_dl/extractor/lecture2go.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index fd115ff54..0075b8a2e 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class Lecture2GoIE(InfoExtractor):
@@ -22,12 +25,26 @@ class Lecture2GoIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
-        video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
+
+        formats = []
+        for url in set(re.findall(r'"src","([^"]+)"', webpage)):
+            ext = determine_ext(url)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(url, video_id))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(url, video_id))
+            else:
+                formats.append({
+                    'url': url,
+                })
+
+        self._sort_formats(formats)
+
         creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'creator': creator
         }

From 1e124295644e3760cd457bb1a6ae717e1cb2c0fc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:05:14 +0800
Subject: [PATCH 1339/1515] [lecture2go] Update _TEST

---
 youtube_dl/extractor/lecture2go.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 0075b8a2e..d0e9416f5 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -11,12 +11,12 @@ class Lecture2GoIE(InfoExtractor):
     _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
-        'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
+        'md5': 'ac02b570883020d208d405d5a3fd2f7f',
         'info_dict': {
             'id': '17473',
-            'ext': 'mp4',
-            'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4',
-            'title': '2 - Endliche Automaten und reguläre Sprachen'
+            'ext': 'flv',
+            'title': '2 - Endliche Automaten und reguläre Sprachen',
+            'creator': 'Frank Heitmann',
         }
     }
 

From 9c29bc69f7d6365835f495dff10f3c5f49671a55 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:15:22 +0800
Subject: [PATCH 1340/1515] [utils] Improve parse_duration

Now dots are parsed. For example '87 Min.'
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e13e11b59..65692a9fb 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -324,6 +324,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('02:03:04'), 7384)
         self.assertEqual(parse_duration('01:02:03:04'), 93784)
         self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
+        self.assertEqual(parse_duration('87 Min.'), 5220)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 942f76d24..ae813099d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1309,10 +1309,10 @@ def parse_duration(s):
     m = re.match(
         r'''(?ix)(?:P?T)?
         (?:
-            (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
+            (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
-            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*|
+            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
             (?:
                 (?:
                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?

From e9c6deffee26db40992293b3055df31804ca7e12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:22:19 +0800
Subject: [PATCH 1341/1515] [lecture2go] Add more metadata fields

---
 youtube_dl/extractor/lecture2go.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index d0e9416f5..a2f9d5c54 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    parse_duration,
+    int_or_none,
+)
 
 
 class Lecture2GoIE(InfoExtractor):
@@ -17,6 +21,7 @@ class Lecture2GoIE(InfoExtractor):
             'ext': 'flv',
             'title': '2 - Endliche Automaten und reguläre Sprachen',
             'creator': 'Frank Heitmann',
+            'duration': 5220,
         }
     }
 
@@ -41,10 +46,16 @@ class Lecture2GoIE(InfoExtractor):
         self._sort_formats(formats)
 
         creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
+        duration = parse_duration(self._html_search_regex(
+            r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'Views:\s*</em>\s*<em[^>]+>(\d+)</em>', webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'creator': creator
+            'creator': creator,
+            'duration': duration,
+            'view_count': view_count,
         }

From 40101dc311909523852a88ba69df76be9b6bc920 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:25:32 +0800
Subject: [PATCH 1342/1515] [lecture2go] Make optional fields non-fatal

---
 youtube_dl/extractor/lecture2go.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index a2f9d5c54..40a3d2346 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -45,7 +45,8 @@ class Lecture2GoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
+        creator = self._html_search_regex(
+            r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator', fatal=False)
         duration = parse_duration(self._html_search_regex(
             r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(

From d609edf4f14e56e3dbb3ffc0be057585c0533666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 22:49:00 +0600
Subject: [PATCH 1343/1515] [udemy] Handle already-logged-in scenario (Closes
 #6327)

---
 youtube_dl/extractor/udemy.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e2bab52fe..4a0eaf65f 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -77,7 +77,11 @@ class UdemyIE(InfoExtractor):
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')
 
-        if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
+        def is_logged(webpage):
+            return any(p in webpage for p in ['href="https://www.udemy.com/user/logout/', '>Logout<'])
+
+        # already logged in
+        if is_logged(login_popup):
             return
 
         login_form = self._form_hidden_inputs('login-form', login_popup)
@@ -95,8 +99,7 @@ class UdemyIE(InfoExtractor):
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if all(logout_pattern not in response
-               for logout_pattern in ['href="https://www.udemy.com/user/logout/', '>Logout<']):
+        if not is_logged(response):
             error = self._html_search_regex(
                 r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                 response, 'error message', default=None)

From ce1bafdce9f7f9e19c172b319a5ae75d1c85759b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 23:49:08 +0600
Subject: [PATCH 1344/1515] [pbs] Clean up title construction rationale

---
 youtube_dl/extractor/pbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ccbe2a9f3..ba6527b80 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -247,8 +247,8 @@ class PBSIE(InfoExtractor):
                 'url': closed_captions_url,
             }]
 
-        # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
-        # 'Full Episode', 'Episode 5', etc. prepend program->title
+        # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc)
+        # Try turning it to 'program - title' naming scheme if possible
         alt_title = info.get('program', {}).get('title')
         if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])

From c7620992d20973320fddb85f936f5dc602d1d708 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 23:49:55 +0600
Subject: [PATCH 1345/1515] [pbs] No need to escape colon

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ba6527b80..a53479aad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -251,7 +251,7 @@ class PBSIE(InfoExtractor):
         # Try turning it to 'program - title' naming scheme if possible
         alt_title = info.get('program', {}).get('title')
         if alt_title:
-            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
+            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title'])
 
         return {
             'id': video_id,

From f79ebf09a24f9bf0e91d0dc26782eeea02f10ad1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 01:56:55 +0800
Subject: [PATCH 1346/1515] Credit @nichdu for Lecture2Go extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 373e05c9f..e75e9885d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -134,3 +134,4 @@ Aurélio A. Heckert
 Bernhard Minks
 sceext
 Zach Bruggeman
+Tjark Saul

From 948199deac4d9ec6ba6bc4359ad43db7951a1c53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 00:30:48 +0600
Subject: [PATCH 1347/1515] [tagesschau] Relax _VALID_URL and simplify

---
 youtube_dl/extractor/tagesschau.py | 62 ++++++++----------------------
 1 file changed, 17 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 636607db5..b84892364 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
@@ -29,57 +29,31 @@ class TagesschauIE(InfoExtractor):
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
-        }
+        },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
-        'md5': '90757268b49ef56deae90c7b48928d58',
-        'info_dict': {
-            'id': '3771',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
-        'md5': '6e3ebdc75e8d67da966a8d06721eda71',
-        'info_dict': {
-            'id': '3827',
-            'ext': 'mp4',
-            'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66',
-            'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
-        'md5': '8a8875a568f0a5ae5ceef93c501a225f',
-        'info_dict': {
-            'id': '3475',
-            'ext': 'mp4',
-            'description': 'md5:ed149f5649cda3dac86813a9d777e131',
-            'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html',
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
-        'md5': 'be4d6f0421f2acd8abe25ea29f6f015b',
-        'info_dict': {
-            'id': '959',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
-        'md5': '42e3757018d9908581481a80cc1806da',
-        'info_dict': {
-            'id': '3299',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
+        'only_matching': True,
     }]
 
     _FORMATS = {
@@ -152,11 +126,9 @@ class TagesschauIE(InfoExtractor):
             thumbnail_fn = self._search_regex(
                 r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
                 webpage, 'thumbnail', fatal=False)
-            # there are some videos without description
-            description = ""
             description = self._html_search_regex(
                 r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', fatal=False, default=None)
+                webpage, 'description', default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From a47b602b0877dcde1b795bf53bfe3629c6595870 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 01:01:04 +0600
Subject: [PATCH 1348/1515] [tagesschau] Add support for audio

---
 youtube_dl/extractor/tagesschau.py | 37 +++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index b84892364..7fd0ba987 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -30,6 +30,16 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
+        'md5': 'aef45de271c4bf0a5db834aa40bf774c',
+        'info_dict': {
+            'id': '18407',
+            'ext': 'mp3',
+            'title': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'description': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'thumbnail': 're:^https?:.*\.jpg$',
+        },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
         'only_matching': True,
@@ -51,9 +61,6 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
         'only_matching': True,
-    }, {
-        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
-        'only_matching': True,
     }]
 
     _FORMATS = {
@@ -73,19 +80,26 @@ class TagesschauIE(InfoExtractor):
             playerpage = self._download_webpage(
                 player_url, display_id, 'Downloading player page')
 
-            medias = re.findall(
-                r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
-                playerpage)
             formats = []
-            for url, ext, res in medias:
+            for media in re.finditer(
+                    r'''(?x)
+                        (?P<q_url>["\'])(?P<url>http://media.+?)(?P=q_url)
+                        ,\s*type:(?P<q_type>["\'])(?P<type>video|audio)/(?P<ext>.+?)(?P=q_type)
+                        (?:,\s*quality:(?P<q_quality>["\'])(?P<quality>.+?)(?P=q_quality))?
+                    ''', playerpage):
+                url = media.group('url')
+                type_ = media.group('type')
+                ext = media.group('ext')
+                res = media.group('quality')
                 f = {
-                    'format_id': res + '_' + ext,
+                    'format_id': '%s_%s' % (res, ext) if res else ext,
                     'url': url,
                     'ext': ext,
+                    'vcodec': 'none' if type_ == 'audio' else None,
                 }
                 f.update(self._FORMATS.get(res, {}))
                 formats.append(f)
-            thumbnail_fn = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+            thumbnail = self._og_search_thumbnail(playerpage)
             title = self._og_search_title(webpage).strip()
             description = self._og_search_description(webpage).strip()
         else:
@@ -123,9 +137,7 @@ class TagesschauIE(InfoExtractor):
                         'filesize_approx': parse_filesize(m.group('filesize_approx')),
                     })
                 formats.append(format)
-            thumbnail_fn = self._search_regex(
-                r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
-                webpage, 'thumbnail', fatal=False)
+            thumbnail = self._og_search_thumbnail(webpage)
             description = self._html_search_regex(
                 r'(?s)<p class="teasertext">(.*?)</p>',
                 webpage, 'description', default=None)
@@ -133,7 +145,6 @@ class TagesschauIE(InfoExtractor):
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
         self._sort_formats(formats)
-        thumbnail = 'http://www.tagesschau.de' + thumbnail_fn
 
         return {
             'id': display_id,

From 3e214851a49992b010ae90ef6dbaed11f70ceb3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 01:01:35 +0600
Subject: [PATCH 1349/1515] [tagesschau] Improve tests

---
 youtube_dl/extractor/tagesschau.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 7fd0ba987..1a7ea98d2 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -18,7 +18,7 @@ class TagesschauIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
             'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
@@ -28,7 +28,7 @@ class TagesschauIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',

From c59b61c0da8fc0fb73d745de59122e290a33b122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 02:13:51 +0600
Subject: [PATCH 1350/1515] [viki] Fix height (Closes #6333)

---
 youtube_dl/extractor/viki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 51cdc6b65..3a7e9a0f2 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -242,8 +242,8 @@ class VikiIE(VikiBaseIE):
 
         formats = []
         for format_id, stream_dict in streams.items():
-            height = self._search_regex(
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
                     formats = self._extract_m3u8_formats(

From 61be92e26a16e24a702c9c65b46f15c375726345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 03:04:23 +0600
Subject: [PATCH 1351/1515] [prosiebensat1] Recognize DRM protected videos
 (#6334)

---
 youtube_dl/extractor/prosiebensat1.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index fec008ce7..0739234c6 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    ExtractorError,
     determine_ext,
     int_or_none,
     unified_strdate,
@@ -224,10 +225,13 @@ class ProSiebenSat1IE(InfoExtractor):
             'ids': clip_id,
         })
 
-        videos = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')
+        video = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')[0]
 
-        duration = float(videos[0]['duration'])
-        source_ids = [source['id'] for source in videos[0]['sources']]
+        if video.get('is_protected') is True:
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        duration = float(video['duration'])
+        source_ids = [source['id'] for source in video['sources']]
         source_ids_str = ','.join(map(str, source_ids))
 
         g = '01!8d8F_)r9]4s[qeuXfP%'

From 993df6bc228ca540bba45e0f5b6041ac9440d695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 03:05:16 +0600
Subject: [PATCH 1352/1515] [prosiebensat1] Modernize

---
 youtube_dl/extractor/prosiebensat1.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 0739234c6..effcf1db3 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     determine_ext,
+    float_or_none,
     int_or_none,
     unified_strdate,
 )
@@ -230,7 +231,7 @@ class ProSiebenSat1IE(InfoExtractor):
         if video.get('is_protected') is True:
             raise ExtractorError('This video is DRM protected.', expected=True)
 
-        duration = float(video['duration'])
+        duration = float_or_none(video.get('duration'))
         source_ids = [source['id'] for source in video['sources']]
         source_ids_str = ','.join(map(str, source_ids))
 

From ca4456eda8f49ed2f939c7ae5cfe55b45777217c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 23:23:38 +0200
Subject: [PATCH 1353/1515] [tagesschau] Remove unused import

---
 youtube_dl/extractor/tagesschau.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 1a7ea98d2..73e7657d4 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize, ExtractorError
+from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):

From 87dc451108b278eb5e5600a0159bdbd337272392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:35:26 +0200
Subject: [PATCH 1354/1515] [youtube] Don't use the DASH manifest from
 'get_video_info' if 'use_cipher_signature' is True (#5118)

Currently they give a 403 Forbidden error.
---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d8b31f98..323681960 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -967,7 +967,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         video_id, note=False,
                         errnote='unable to download video info webpage')
                     get_video_info = compat_parse_qs(video_info_webpage)
-                    add_dash_mpd(get_video_info)
+                    if get_video_info.get('use_cipher_signature') != ['True']:
+                        add_dash_mpd(get_video_info)
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:

From b37317d8b0391651b396fc2dc4e35b59e52fddf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 23:33:49 +0200
Subject: [PATCH 1355/1515] [generic] Unescape HTML escape sequences in
 redirect urls (fixes #6311)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cd133a10c..6d2efb22e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1702,7 +1702,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = compat_urlparse.urljoin(url, found.group(1))
+                new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
                 self.report_following_redirect(new_url)
                 return {
                     '_type': 'url',

From 41597d9bed9eaa5e55d5bb572f2ec3f5a312d392 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 13:39:19 +0800
Subject: [PATCH 1356/1515] [viki] Fix description extraction (closes #6339)

---
 youtube_dl/extractor/viki.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 3a7e9a0f2..e987badbd 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -173,6 +173,19 @@ class VikiIE(VikiBaseIE):
     }, {
         'url': 'http://www.viki.com/player/44699v',
         'only_matching': True,
+    }, {
+        # non-English description
+        'url': 'http://www.viki.com/videos/158036v-love-in-magic',
+        'md5': '1713ae35df5a521b31f6dc40730e7c9c',
+        'info_dict': {
+            'id': '158036v',
+            'ext': 'mp4',
+            'uploader': 'I Planet Entertainment',
+            'upload_date': '20111122',
+            'timestamp': 1321985454,
+            'description': 'md5:44b1e46619df3a072294645c770cef36',
+            'title': 'Love In Magic',
+        },
     }]
 
     def _real_extract(self, url):
@@ -192,8 +205,12 @@ class VikiIE(VikiBaseIE):
                 container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                 title = '%s - %s' % (container_title, title)
 
-        descriptions = video.get('descriptions')
-        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
+        descriptions = video.get('descriptions', {})
+        description = descriptions.get('en')
+        if description is None:
+            filtered_descriptions = list(filter(None, [descriptions.get(k) for k in titles.keys()]))
+            if filtered_descriptions:
+                description = filtered_descriptions[0]
 
         duration = int_or_none(video.get('duration'))
         timestamp = parse_iso8601(video.get('created_at'))

From b73b14f72c8240466b5bded9fb891697549b89ec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 14:02:19 +0800
Subject: [PATCH 1357/1515] [viki] Rewrite dict selection codes

---
 youtube_dl/extractor/viki.py | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index e987badbd..ddbd395c8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -88,6 +88,14 @@ class VikiBaseIE(InfoExtractor):
         if not self._token:
             self.report_warning('Unable to get session token, login has probably failed')
 
+    @staticmethod
+    def dict_selection(dict_obj, preferred_key):
+        if preferred_key in dict_obj:
+            return dict_obj.get(preferred_key)
+
+        filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
+        return filtered_dict[0] if filtered_dict else None
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
@@ -194,23 +202,14 @@ class VikiIE(VikiBaseIE):
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
-        title = None
-        titles = video.get('titles')
-        if titles:
-            title = titles.get('en') or titles[titles.keys()[0]]
+        title = self.dict_selection(video.get('titles', {}), 'en')
         if not title:
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
-            container_titles = video.get('container', {}).get('titles')
-            if container_titles:
-                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
-                title = '%s - %s' % (container_title, title)
+            container_titles = video.get('container', {}).get('titles', {})
+            container_title = self.dict_selection(container_titles, 'en')
+            title = '%s - %s' % (container_title, title)
 
-        descriptions = video.get('descriptions', {})
-        description = descriptions.get('en')
-        if description is None:
-            filtered_descriptions = list(filter(None, [descriptions.get(k) for k in titles.keys()]))
-            if filtered_descriptions:
-                description = filtered_descriptions[0]
+        description = self.dict_selection(video.get('descriptions', {}), 'en')
 
         duration = int_or_none(video.get('duration'))
         timestamp = parse_iso8601(video.get('created_at'))
@@ -316,11 +315,9 @@ class VikiChannelIE(VikiBaseIE):
             'containers/%s.json' % channel_id, channel_id,
             'Downloading channel JSON')
 
-        titles = channel['titles']
-        title = titles.get('en') or titles[titles.keys()[0]]
+        title = self.dict_selection(channel['titles'], 'en')
 
-        descriptions = channel['descriptions']
-        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
+        description = self.dict_selection(channel['descriptions'], 'en')
 
         entries = []
         for video_type in ('episodes', 'clips', 'movies'):

From 59db9f80187c4dc3768f8656ede281347a805e1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 12:09:30 +0200
Subject: [PATCH 1358/1515] [downloader/dash] Improve 'combine_url' (fixes
 #6341)

In some videos the base_url already ends with '/'.
---
 youtube_dl/downloader/dash.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index a4685d307..8b6fa2753 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -37,7 +37,7 @@ class DashSegmentsFD(FileDownloader):
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
                 return target_url
-            return '%s/%s' % (base_url, target_url)
+            return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         with open(tmpfilename, 'wb') as outf:
             append_url_to_file(

From 53b8247cb5b5ac4a7822c82b94ec2f1221a40625 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Thu, 23 Jul 2015 01:38:55 -0500
Subject: [PATCH 1359/1515] NationalGeographic._VALID_URL: work site-wide

Closes #6343.
---
 youtube_dl/extractor/nationalgeographic.py | 35 +++++++++++++++-------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index f793b72f5..6fc9e7b05 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -8,18 +8,30 @@ from ..utils import (
 
 
 class NationalGeographicIE(InfoExtractor):
-    _VALID_URL = r'http://video\.nationalgeographic\.com/video/.*?'
+    _VALID_URL = r'http://video\.nationalgeographic\.com/.*?'
 
-    _TEST = {
-        'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
-        'info_dict': {
-            'id': '4DmDACA6Qtk_',
-            'ext': 'flv',
-            'title': 'Mating Crabs Busted by Sharks',
-            'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+    _TESTS = [
+        {
+            'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+            'info_dict': {
+                'id': '4DmDACA6Qtk_',
+                'ext': 'flv',
+                'title': 'Mating Crabs Busted by Sharks',
+                'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+            },
+            'add_ie': ['ThePlatform'],
         },
-        'add_ie': ['ThePlatform'],
-    }
+        {
+            'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
+            'info_dict': {
+                'id': '_JeBD_D7PlS5',
+                'ext': 'flv',
+                'title': 'The Real Jaws',
+                'description': 'md5:8d3e09d9d53a85cd397b4b21b2c77be6',
+            },
+            'add_ie': ['ThePlatform'],
+        },
+    ]
 
     def _real_extract(self, url):
         name = url_basename(url)
@@ -37,5 +49,6 @@ class NationalGeographicIE(InfoExtractor):
 
         return self.url_result(smuggle_url(
             'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
-            # For some reason, the normal links don't work and we must force the use of f4m
+            # For some reason, the normal links don't work and we must force
+            # the use of f4m
             {'force_smil_url': True}))

From 297a564beeb20ca8b00d94f5707532110631f409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 13:20:21 +0200
Subject: [PATCH 1360/1515] [youtube] Extract end_time

---
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e8751877..1272834c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -185,6 +185,8 @@ class InfoExtractor(object):
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the url.
+    end_time:       Time in seconds where the reproduction should end, as
+                    specified in the url.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index afbd34f4a..117ef2e77 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -319,7 +319,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -332,6 +332,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
+                'end_time': 9,
             }
         },
         {
@@ -893,12 +894,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else 'https')
 
         start_time = None
+        end_time = None
         parsed_url = compat_urllib_parse_urlparse(url)
         for component in [parsed_url.fragment, parsed_url.query]:
             query = compat_parse_qs(component)
-            if 't' in query:
+            if start_time is None and 't' in query:
                 start_time = parse_duration(query['t'][0])
-                break
+            if end_time is None and 'end' in query:
+                end_time = parse_duration(query['end'][0])
 
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
@@ -1267,6 +1270,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'formats': formats,
             'is_live': is_live,
             'start_time': start_time,
+            'end_time': end_time,
         }
 
 

From 2929fa0e79dfd3a1366e7e23eb4344bc93dd3a10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 13:21:18 +0200
Subject: [PATCH 1361/1515] [youtube] Also look into the 'start' field for
 start_time

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 117ef2e77..462d244d8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -900,6 +900,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             query = compat_parse_qs(component)
             if start_time is None and 't' in query:
                 start_time = parse_duration(query['t'][0])
+            if start_time is None and 'start' in query:
+                start_time = parse_duration(query['start'][0])
             if end_time is None and 'end' in query:
                 end_time = parse_duration(query['end'][0])
 

From d3f007af183a0f5ed278602128e6bba3cc1350b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 22:04:47 +0600
Subject: [PATCH 1362/1515] [daylimotion] Adapt to player v5 and modernize
 (Closes #6151, closes #6250)

---
 youtube_dl/extractor/dailymotion.py | 160 ++++++++++++++++++++--------
 1 file changed, 117 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 1a41c0db1..8fcae7402 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -13,8 +13,10 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    determine_ext,
     int_or_none,
     orderedSet,
+    parse_iso8601,
     str_to_int,
     unescapeHTML,
 )
@@ -28,10 +30,12 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
+    def _download_webpage_no_ff(self, url, *args, **kwargs):
+        request = self._build_request(url)
+        return self._download_webpage(request, *args, **kwargs)
+
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    """Information Extractor for Dailymotion"""
-
     _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
@@ -50,10 +54,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'info_dict': {
                 'id': 'x2iuewm',
                 'ext': 'mp4',
-                'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
-                'upload_date': '20150306',
+                'description': 'Several come bundled with the Steam Controller.',
+                'thumbnail': 're:^https?:.*\.(?:jpg|png)$',
                 'duration': 74,
+                'timestamp': 1425657362,
+                'upload_date': '20150306',
+                'uploader': 'IGN',
+                'uploader_id': 'xijv66',
+                'age_limit': 0,
+                'view_count': int,
+                'comment_count': int,
             }
         },
         # Vevo video
@@ -87,38 +98,106 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.dailymotion.com/video/%s' % video_id
 
-        # Retrieve video webpage to extract further information
-        request = self._build_request(url)
-        webpage = self._download_webpage(request, video_id)
-
-        # Extract URL, uploader and title from webpage
-        self.report_extraction(video_id)
-
-        # It may just embed a vevo video:
-        m_vevo = re.search(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
-            webpage)
-        if m_vevo is not None:
-            vevo_id = m_vevo.group('id')
-            self.to_screen('Vevo video detected: %s' % vevo_id)
-            return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+        webpage = self._download_webpage_no_ff(
+            'https://www.dailymotion.com/video/%s' % video_id, video_id)
 
         age_limit = self._rta_search(webpage)
 
-        video_upload_date = None
-        mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
-        if mobj is not None:
-            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
+
+        view_count = str_to_int(self._search_regex(
+            [r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:(\d+)"',
+             r'video_views_count[^>]+>\s+([\d\.,]+)'],
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
+            webpage, 'comment count', fatal=False))
+
+        player_v5 = self._search_regex(
+            r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+            webpage, 'player v5', default=None)
+        if player_v5:
+            player = self._parse_json(player_v5, video_id)
+            metadata = player['metadata']
+            formats = []
+            for quality, media_list in metadata['qualities'].items():
+                for media in media_list:
+                    media_url = media.get('url')
+                    if not media_url:
+                        continue
+                    type_ = media.get('type')
+                    if type_ == 'application/vnd.lumberjack.manifest':
+                        continue
+                    if type_ == 'application/x-mpegURL' or determine_ext(media_url) == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls'))
+                    else:
+                        f = {
+                            'url': media_url,
+                            'format_id': quality,
+                        }
+                        m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
+                        if m:
+                            f.update({
+                                'width': int(m.group('width')),
+                                'height': int(m.group('height')),
+                            })
+                        formats.append(f)
+            self._sort_formats(formats)
+
+            title = metadata['title']
+            duration = int_or_none(metadata.get('duration'))
+            timestamp = int_or_none(metadata.get('created_time'))
+            thumbnail = metadata.get('poster_url')
+            uploader = metadata.get('owner', {}).get('screenname')
+            uploader_id = metadata.get('owner', {}).get('id')
+
+            subtitles = {}
+            for subtitle_lang, subtitle in metadata.get('subtitles', {}).get('data', {}).items():
+                subtitles[subtitle_lang] = [{
+                    'ext': determine_ext(subtitle_url),
+                    'url': subtitle_url,
+                } for subtitle_url in subtitle.get('urls', [])]
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'timestamp': timestamp,
+                'uploader': uploader,
+                'uploader_id': uploader_id,
+                'age_limit': age_limit,
+                'view_count': view_count,
+                'comment_count': comment_count,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # vevo embed
+        vevo_id = self._search_regex(
+            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
+            webpage, 'vevo embed', default=None)
+        if vevo_id:
+            return self.url_result('vevo:%s' % vevo_id, 'Vevo')
+
+        # fallback old player
+        embed_page = self._download_webpage_no_ff(
+            'https://www.dailymotion.com/embed/video/%s' % video_id,
+            video_id, 'Downloading embed page')
+
+        timestamp = parse_iso8601(self._html_search_meta(
+            'video:release_date', webpage, 'upload date'))
+
+        info = self._parse_json(
+            self._search_regex(
+                r'var info = ({.*?}),$', embed_page,
+                'video info', flags=re.MULTILINE),
+            video_id)
 
-        embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
-        embed_request = self._build_request(embed_url)
-        embed_page = self._download_webpage(
-            embed_request, video_id, 'Downloading embed page')
-        info = self._search_regex(r'var info = ({.*?}),$', embed_page,
-                                  'video info', flags=re.MULTILINE)
-        info = json.loads(info)
         if info.get('error') is not None:
             msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
             raise ExtractorError(msg, expected=True)
@@ -139,16 +218,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     'width': width,
                     'height': height,
                 })
-        if not formats:
-            raise ExtractorError('Unable to extract video URL')
+        self._sort_formats(formats)
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, webpage)
 
-        view_count = str_to_int(self._search_regex(
-            r'video_views_count[^>]+>\s+([\d\.,]+)',
-            webpage, 'view count', fatal=False))
-
         title = self._og_search_title(webpage, default=None)
         if title is None:
             title = self._html_search_regex(
@@ -159,8 +233,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'id': video_id,
             'formats': formats,
             'uploader': info['owner.screenname'],
-            'upload_date': video_upload_date,
+            'timestamp': timestamp,
             'title': title,
+            'description': description,
             'subtitles': video_subtitles,
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
@@ -201,9 +276,9 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     def _extract_entries(self, id):
         video_ids = []
         for pagenum in itertools.count(1):
-            request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
-            webpage = self._download_webpage(request,
-                                             id, 'Downloading page %s' % pagenum)
+            webpage = self._download_webpage_no_ff(
+                self._PAGE_TEMPLATE % (id, pagenum),
+                id, 'Downloading page %s' % pagenum)
 
             video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
 
@@ -286,8 +361,7 @@ class DailymotionCloudIE(DailymotionBaseInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = self._build_request(url)
-        webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage_no_ff(url, video_id)
 
         title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
 

From ba911137fa53737b663cb2879d357bc8a35ab558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 22:50:59 +0600
Subject: [PATCH 1363/1515] [rts] Add support for articles with videos on rhs
 (Closes #6332)

---
 youtube_dl/extractor/rts.py | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 9fbe239d8..12639f08b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -19,7 +19,16 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        rts:(?P<rts_id>\d+)|
+                        https?://
+                            (?:www\.)?rts\.ch/
+                            (?:
+                                (?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|
+                                play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+)
+                            )
+                    )'''
 
     _TESTS = [
         {
@@ -122,6 +131,15 @@ class RTSIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        {
+            # article with videos on rhs
+            'url': 'http://www.rts.ch/sport/hockey/6693917-hockey-davos-decroche-son-31e-titre-de-champion-de-suisse.html',
+            'info_dict': {
+                'id': '6693917',
+                'title': 'Hockey: Davos décroche son 31e titre de champion de Suisse',
+            },
+            'playlist_mincount': 5,
+        },
         {
             'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
             'only_matching': True,
@@ -130,7 +148,7 @@ class RTSIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('id') or m.group('id_new')
+        video_id = m.group('rts_id') or m.group('id') or m.group('id_new')
         display_id = m.group('display_id') or m.group('display_id_new')
 
         def download_json(internal_id):
@@ -143,6 +161,15 @@ class RTSIE(InfoExtractor):
         # video_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
             page = self._download_webpage(url, display_id)
+
+            # article with videos on rhs
+            videos = re.findall(
+                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:rts:video:(\d+)"',
+                page)
+            if videos:
+                entries = [self.url_result('rts:%s' % video_urn, 'RTS') for video_urn in videos]
+                return self.playlist_result(entries, video_id, self._og_search_title(page))
+
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')

From a8b7b26068fa8de9983ffef995ea6cd4fd3ce90a Mon Sep 17 00:00:00 2001
From: corone17 <reino@degeelebosch.nl>
Date: Wed, 22 Jul 2015 22:38:04 +0200
Subject: [PATCH 1364/1515] Undo adaptive -> flash workaround

For a couple of days now rtlXL's non-DRM adaptive (m3u8) streams don't work anymore. By undoing the adaptive -> flash workaround youtube-dl returns the DRM streams again. Too bad for foreigners (geoblock).
The progressive streams still work fine.
---
 youtube_dl/extractor/rtlnl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e0c530d64..518f7a7b4 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -86,12 +86,12 @@ class RtlNlIE(InfoExtractor):
         # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
         # this adaptive -> flash workaround is not required in general, but it also
         # allows bypassing georestriction therefore is retained for now.
-        videopath = material['videopath'].replace('/adaptive/', '/flash/')
+        videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
-        video_urlpart = videopath.split('/flash/')[1][:-5]
+        video_urlpart = videopath.split('/adaptive/')[1][:-5]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([

From a9e8f60ef686eeeb3d31f2e5211ce176dc8f5298 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 23:20:24 +0600
Subject: [PATCH 1365/1515] [rtlnl] Update unencrypted streams comment (#6337)

---
 youtube_dl/extractor/rtlnl.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 518f7a7b4..543d94417 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -82,10 +82,15 @@ class RtlNlIE(InfoExtractor):
 
         meta = info.get('meta', {})
 
-        # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
-        # this adaptive -> flash workaround is not required in general, but it also
-        # allows bypassing georestriction therefore is retained for now.
+        # m3u8 streams are encrypted and may not be handled properly by older ffmpeg/avconv.
+        # To workaround this previously adaptive -> flash trick was used to obtain
+        # unencrypted m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
+        # and bypass georestrictions as well.
+        # Currently, unencrypted m3u8 playlists are (intentionally?) invalid and therefore
+        # unusable albeit can be fixed by simple string replacement (see
+        # https://github.com/rg3/youtube-dl/pull/6337)
+        # Since recent ffmpeg and avconv handle encrypted streams just fine encrypted
+        # streams are used now.
         videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 

From 10952eb2cf2c86c841e46690a3bfa6a39c82d67d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 23:37:45 +0600
Subject: [PATCH 1366/1515] [extractor/common] Consistent URL spelling

---
 youtube_dl/extractor/common.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1272834c5..14b9b4fe2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -65,7 +65,7 @@ class InfoExtractor(object):
 
                     Potential fields:
                     * url        Mandatory. The URL of the video file
-                    * ext        Will be calculated from url if missing
+                    * ext        Will be calculated from URL if missing
                     * format     A human-readable description of the format
                                  ("mp4 container with h264/opus").
                                  Calculated from the format_id, width, height.
@@ -155,7 +155,7 @@ class InfoExtractor(object):
                     lower to higher preference, each element is a dictionary
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
-                        * "url": A url pointing to the subtitles file
+                        * "url": A URL pointing to the subtitles file
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
     duration:       Length of the video in seconds, as an integer.
@@ -176,7 +176,7 @@ class InfoExtractor(object):
                                      Set to "root" to indicate that this is a
                                      comment to the original video.
     age_limit:      Age restriction for the video, as an integer (years)
-    webpage_url:    The url to the video webpage, if given to youtube-dl it
+    webpage_url:    The URL to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
@@ -184,9 +184,9 @@ class InfoExtractor(object):
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
-                    specified in the url.
+                    specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
-                    specified in the url.
+                    specified in the URL.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -505,7 +505,7 @@ class InfoExtractor(object):
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):
-        """Returns a url that points to a page that should be processed"""
+        """Returns a URL that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
                       'url': url,
@@ -639,7 +639,7 @@ class InfoExtractor(object):
         return unescapeHTML(escaped)
 
     def _og_search_thumbnail(self, html, **kargs):
-        return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
+        return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
 
     def _og_search_description(self, html, **kargs):
         return self._og_search_property('description', html, fatal=False, **kargs)
@@ -1120,7 +1120,7 @@ class InfoExtractor(object):
 class SearchInfoExtractor(InfoExtractor):
     """
     Base class for paged search queries extractors.
-    They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
+    They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
     Instances should define _SEARCH_KEY and _MAX_RESULTS.
     """
 

From 660f9459dabfc01c7235b4cc57f9c561de522e07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 00:17:36 +0600
Subject: [PATCH 1367/1515] [canalplus] Fix m3u8 videos extension

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 699b4f7d0..8671236dd 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -106,7 +106,7 @@ class CanalplusIE(InfoExtractor):
                 continue
             format_id = fmt.tag
             if format_id == 'HLS':
-                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
+                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4')
                 for fmt in hls_formats:
                     fmt['preference'] = preference(format_id)
                 formats.extend(hls_formats)

From f3f0b8e4030a07e07385afe6ebb4485ac33e5357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 00:21:13 +0600
Subject: [PATCH 1368/1515] [canalplus] Modernize

---
 youtube_dl/extractor/canalplus.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 8671236dd..57e0cda2c 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -106,15 +106,11 @@ class CanalplusIE(InfoExtractor):
                 continue
             format_id = fmt.tag
             if format_id == 'HLS':
-                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4')
-                for fmt in hls_formats:
-                    fmt['preference'] = preference(format_id)
-                formats.extend(hls_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', preference=preference(format_id)))
             elif format_id == 'HDS':
-                hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
-                for fmt in hds_formats:
-                    fmt['preference'] = preference(format_id)
-                formats.extend(hds_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url + '?hdcore=2.11.3', video_id, preference=preference(format_id)))
             else:
                 formats.append({
                     'url': format_url,

From d97f5cd795b0bd8b6bd636b899bdbc8b10d5907b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 02:56:54 +0600
Subject: [PATCH 1369/1515] [bbccouk] Make more robust (Closes #6345)

---
 youtube_dl/extractor/bbccouk.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 5825d2867..b2e5f7418 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -210,12 +210,12 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_video(self, media, programme_id):
         formats = []
-        vbr = int(media.get('bitrate'))
+        vbr = int_or_none(media.get('bitrate'))
         vcodec = media.get('encoding')
         service = media.get('service')
-        width = int(media.get('width'))
-        height = int(media.get('height'))
-        file_size = int(media.get('media_file_size'))
+        width = int_or_none(media.get('width'))
+        height = int_or_none(media.get('height'))
+        file_size = int_or_none(media.get('media_file_size'))
         for connection in self._extract_connections(media):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
@@ -232,7 +232,7 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_audio(self, media, programme_id):
         formats = []
-        abr = int(media.get('bitrate'))
+        abr = int_or_none(media.get('bitrate'))
         acodec = media.get('encoding')
         service = media.get('service')
         for connection in self._extract_connections(media):
@@ -300,7 +300,7 @@ class BBCCoUkIE(InfoExtractor):
                     if kind != 'programme' and kind != 'radioProgramme':
                         continue
                     programme_id = item.get('vpid')
-                    duration = int(item.get('duration'))
+                    duration = int_or_none(item.get('duration'))
                     formats, subtitles = self._download_media_selector(programme_id)
                 return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
@@ -332,7 +332,7 @@ class BBCCoUkIE(InfoExtractor):
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
             programme_id = item.get('identifier')
-            duration = int(item.get('duration'))
+            duration = int_or_none(item.get('duration'))
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles

From 2b2ee140c3c6f08b4078cc6a5a289e5e74bec2b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 03:40:24 +0600
Subject: [PATCH 1370/1515] [dailymotion:user] Fix _VALID_URL (Closes #6346)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 8fcae7402..5a4987772 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -302,7 +302,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1a117a77287e7dbd4d92f29062dabcf4efb86cb5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 24 Jul 2015 12:00:20 +0100
Subject: [PATCH 1371/1515] [clipfish] extract mp4 video link

---
 youtube_dl/extractor/clipfish.py | 37 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index a5c3cb7c6..09dfaac60 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-import time
-import xml.etree.ElementTree
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_duration,
+    int_or_none,
+    js_to_json,
+    determine_ext,
 )
 
 
@@ -17,37 +15,40 @@ class ClipfishIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
     _TEST = {
         'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
-        'md5': '2521cd644e862936cf2e698206e47385',
+        'md5': '79bc922f3e8a9097b3d68a93780fd475',
         'info_dict': {
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
             'duration': 82,
-        },
-        'skip': 'Blocked in the US'
+        }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_info = self._parse_json(
+            js_to_json(self._html_search_regex('var videoObject = ({[^}]+?})', webpage, 'videoObject')),
+            video_id
+        )
+        info_url = self._parse_json(
+            js_to_json(self._html_search_regex('var globalFlashvars = ({[^}]+?})', webpage, 'globalFlashvars')),
+            video_id
+        )['data']
 
-        info_url = ('http://www.clipfish.de/devxml/videoinfo/%s?ts=%d' %
-                    (video_id, int(time.time())))
         doc = self._download_xml(
             info_url, video_id, note='Downloading info page')
         title = doc.find('title').text
         video_url = doc.find('filename').text
-        if video_url is None:
-            xml_bytes = xml.etree.ElementTree.tostring(doc)
-            raise ExtractorError('Cannot find video URL in document %r' %
-                                 xml_bytes)
         thumbnail = doc.find('imageurl').text
-        duration = parse_duration(doc.find('duration').text)
+        duration = int_or_none(video_info['length'])
+        formats = [{'url': video_info['videourl']},{'url': video_url}]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
         }

From 12434026574bcaaaa705c31ef14428cc91a5efad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Jul 2015 21:29:44 +0800
Subject: [PATCH 1372/1515] [dailymotion:playlist] Detect problematic
 redirection (fixes #6347)

---
 youtube_dl/extractor/dailymotion.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 5a4987772..85d945509 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -30,6 +30,10 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
+    def _download_webpage_handle_no_ff(self, url, *args, **kwargs):
+        request = self._build_request(url)
+        return self._download_webpage_handle(request, *args, **kwargs)
+
     def _download_webpage_no_ff(self, url, *args, **kwargs):
         request = self._build_request(url)
         return self._download_webpage(request, *args, **kwargs)
@@ -275,10 +279,17 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
     def _extract_entries(self, id):
         video_ids = []
+        processed_urls = set()
         for pagenum in itertools.count(1):
-            webpage = self._download_webpage_no_ff(
-                self._PAGE_TEMPLATE % (id, pagenum),
-                id, 'Downloading page %s' % pagenum)
+            page_url = self._PAGE_TEMPLATE % (id, pagenum)
+            webpage, urlh = self._download_webpage_handle_no_ff(
+                page_url, id, 'Downloading page %s' % pagenum)
+            if urlh.geturl() in processed_urls:
+                self.report_warning('Stopped at duplicated page %s, which is the same as %s' % (
+                    page_url, urlh.geturl()), id)
+                break
+
+            processed_urls.add(urlh.geturl())
 
             video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
 
@@ -311,6 +322,17 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': 'Rémi Gaillard',
         },
         'playlist_mincount': 100,
+    }, {
+        'url': 'http://www.dailymotion.com/user/UnderProject',
+        'info_dict': {
+            'id': 'UnderProject',
+            'title': 'UnderProject',
+        },
+        'playlist_mincount': 1800,
+        'expected_warnings': [
+            'Stopped at duplicated page',
+        ],
+        'skip': 'Takes too long time',
     }]
 
     def _real_extract(self, url):

From 593ddd851b87eb7ac327a9216800162f07b2acca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 24 Jul 2015 14:46:45 +0100
Subject: [PATCH 1373/1515] [letv] fix height

---
 youtube_dl/extractor/letv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ba2ae8085..a28abb0f0 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -15,6 +15,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
+    int_or_none,
 )
 
 
@@ -134,7 +135,7 @@ class LetvIE(InfoExtractor):
                 }
 
                 if format_id[-1:] == 'p':
-                    url_info_dict['height'] = format_id[:-1]
+                    url_info_dict['height'] = int_or_none(format_id[:-1])
 
                 urls.append(url_info_dict)
 

From 678e436f2e77f1ae3a57c4b5d1fc3d74342ab412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 02:09:34 +0600
Subject: [PATCH 1374/1515] [youtube] Handle empty allowed regions (Closes
 #6351)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 229fc3a0f..4023a6e50 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -993,7 +993,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if 'reason' in video_info:
                 if 'The uploader has not made this video available in your country.' in video_info['reason']:
                     regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
-                    if regions_allowed is not None:
+                    if regions_allowed:
                         raise ExtractorError('YouTube said: This video is available in %s only' % (
                             ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
                             expected=True)

From b14fa8e6874818a3f210b2a67cf53345000defdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 15:47:53 +0600
Subject: [PATCH 1375/1515] [soundcloud:set] Defer download link resolve
 (Closes #6354)

---
 youtube_dl/extractor/soundcloud.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 118ca4832..cdee8e2a3 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -282,9 +282,11 @@ class SoundcloudSetIE(SoundcloudIE):
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]
+
         return {
             '_type': 'playlist',
-            'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
+            'entries': entries,
             'id': '%s' % info['id'],
             'title': info['title'],
         }

From 40a2d17052e9b542eb3c360a0ce067d244e07fb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 15:48:44 +0600
Subject: [PATCH 1376/1515] [soundcloud:playlist] Defer download link resolve

---
 youtube_dl/extractor/soundcloud.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index cdee8e2a3..0a6c9fe72 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -381,9 +381,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
-        entries = [
-            self._extract_info_dict(t, quiet=True, secret_token=token)
-            for t in data['tracks']]
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]
 
         return {
             '_type': 'playlist',

From eab7faa0c1e8511bc91c64347d0dffc28c94f101 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:39:01 +0800
Subject: [PATCH 1377/1515] [ir90tv] Test (?:www\.)? part in _VALID_URL

---
 youtube_dl/extractor/ir90tv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index b79529b1b..880a6e32f 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class Ir90TvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
         'md5': '411dbd94891381960cb9e13daa47a869',
         'info_dict': {
@@ -15,7 +15,10 @@ class Ir90TvIE(InfoExtractor):
             'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 9700cd9097445d598515dc41fb3cb9421403b9b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:42:40 +0800
Subject: [PATCH 1378/1515] [ir90tv] Improve title extraction

---
 youtube_dl/extractor/ir90tv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 880a6e32f..92333c3ea 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import remove_start
 
 
 class Ir90TvIE(InfoExtractor):
@@ -24,8 +25,8 @@ class Ir90TvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
+        title = remove_start(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), '90tv.ir :: ')
 
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')

From 7523647391969f8d747ba0fc178592f7f3d5e453 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:43:07 +0800
Subject: [PATCH 1379/1515] [ir90tv] PEP8

---
 youtube_dl/extractor/ir90tv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 92333c3ea..6418d6178 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -37,6 +37,6 @@ class Ir90TvIE(InfoExtractor):
             'url': video_url,
             'id': video_id,
             'title': title,
-            'video_url' : video_url,
-            'thumbnail' : thumbnail,
+            'video_url': video_url,
+            'thumbnail': thumbnail,
         }

From 2c7c721933e53ece49bee0140d2dad9a8219d6e4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:48:00 +0800
Subject: [PATCH 1380/1515] [ir90tv] Optional fields should be non-fatal

---
 youtube_dl/extractor/ir90tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 6418d6178..214bcd5b5 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -31,7 +31,7 @@ class Ir90TvIE(InfoExtractor):
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
-        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)
 
         return {
             'url': video_url,

From 9afa1770d1a6835bc8fee48dc86cd1a702d1f67a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 20:21:42 +0600
Subject: [PATCH 1381/1515] [bbc] Improve playlist extraction, refactor, expand
 support and document

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/bbc.py      | 375 +++++++++++++++++++++----------
 2 files changed, 259 insertions(+), 121 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bc61cbdc5..d77ed3ba2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -43,7 +43,10 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbc import BBCCoUkIE, BBCNewsIE
+from .bbc import (
+    BBCCoUkIE,
+    BBCIE,
+)
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 86327d8ed..2a0901ee4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1,15 +1,18 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_duration,
+    float_or_none,
     int_or_none,
+    parse_duration,
+    parse_iso8601,
 )
 from ..compat import compat_HTTPError
-import re
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -17,7 +20,7 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
 
     _TESTS = [
         {
@@ -264,16 +267,21 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
+        return self._download_media_selector_url(
+            self._MEDIASELECTOR_URL % programme_id, programme_id)
+
+    def _download_media_selector_url(self, url, programme_id=None):
         try:
             media_selection = self._download_xml(
-                self.mediaselector_url % programme_id,
-                programme_id, 'Downloading media selection XML')
+                url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                 media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
+        return self._process_media_selector(media_selection, programme_id)
 
+    def _process_media_selector(self, media_selection, programme_id):
         formats = []
         subtitles = None
 
@@ -312,10 +320,21 @@ class BBCCoUkIE(InfoExtractor):
                 raise
 
         # fallback to legacy playlist
-        playlist = self._download_xml(
-            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
-            playlist_id, 'Downloading legacy playlist XML')
+        return self._process_legacy_playlist(playlist_id)
 
+    def _process_legacy_playlist_url(self, url, display_id):
+        playlist = self._download_legacy_playlist_url(url, display_id)
+        return self._extract_from_legacy_playlist(playlist, display_id)
+
+    def _process_legacy_playlist(self, playlist_id):
+        return self._process_legacy_playlist_url(
+            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)
+
+    def _download_legacy_playlist_url(self, url, playlist_id=None):
+        return self._download_xml(
+            url, playlist_id, 'Downloading legacy playlist XML')
+
+    def _extract_from_legacy_playlist(self, playlist, playlist_id):
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:
             reason = no_items.get('reason')
@@ -335,8 +354,23 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
-            programme_id = item.get('identifier')
+
+            def get_programme_id(item):
+                def get_from_attributes(item):
+                    for p in('identifier', 'group'):
+                        value = item.get(p)
+                        if value and re.match(r'^[pb][\da-z]{7}$', value):
+                            return value
+                get_from_attributes(item)
+                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                if mediator is not None:
+                    return get_from_attributes(mediator)
+
+            programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
+            # TODO: programme_id can be None and media items can be incorporated right inside
+            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+            # as f4m and m3u8
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles
@@ -383,175 +417,276 @@ class BBCCoUkIE(InfoExtractor):
         }
 
 
-class BBCNewsIE(BBCCoUkIE):
-    IE_NAME = 'bbc.com'
-    IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.com/.+?/(?P<id>[^/]+)$'
+class BBCIE(BBCCoUkIE):
+    IE_NAME = 'bbc'
+    IE_DESC = 'BBC'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # fails with notukerror for some videos
+    #_MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{
+        # article with multiple videos embedded with data-media-meta containing
+        # playlist.sxml, externalId and no direct video links
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
             'id': 'world-europe-32668511',
             'title': 'Russia stages massive WW2 parade despite Western boycott',
+            'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
         },
         'playlist_count': 2,
     }, {
+        # article with multiple videos embedded with data-media-meta (more videos)
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
             'title': 'Farnborough Airshow: Video highlights',
+            'description': 'BBC reports and video highlights at the Farnborough Airshow.',
         },
         'playlist_count': 9,
+        'skip': 'Save time',
     }, {
+        # single video embedded with mediaAssetPage.init()
         'url': 'http://www.bbc.com/news/world-europe-32041533',
-        'note': 'Video',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
+            'timestamp': 1427219242,
             'upload_date': '20150324',
-            'uploader': 'BBC News',
         },
         'params': {
+            # rtmp download
             'skip_download': True,
         }
     }, {
+        # article with single video embedded with data-media-meta containing
+        # direct video links (for now these are extracted) and playlist.xml (with
+        # media items as f4m and m3u8 - currently unsupported)
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
-        'note': 'Video',
         'info_dict': {
-            'id': 'NA',
+            'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
-            'title': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
-            'description': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
+            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
             'duration': 47,
+            'timestamp': 1434397334,
             'upload_date': '20150615',
-            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
         }
     }, {
+        # single video embedded with mediaAssetPage.init() (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
-        'note': 'Video',
         'info_dict': {
-            'id': '39275083',
+            'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
-            'title': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
-            'description': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
+            'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
             'duration': 87,
+            'timestamp': 1434713142,
             'upload_date': '20150619',
-            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video story with digitalData
+        'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
+        'info_dict': {
+            'id': 'p02q6gc4',
+            'ext': 'flv',
+            'title': 'Sri Lanka’s spicy secret',
+            'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
+            'timestamp': 1437674293,
+            'upload_date': '20150723',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video story without digitalData
+        'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
+        'info_dict': {
+            'id': 'p018zqqg',
+            'ext': 'flv',
+            'title': 'Hyundai Santa Fe Sport: Rock star',
+            'description': 'md5:b042a26142c4154a6e472933cf20793d',
+            'timestamp': 1368473503,
+            'upload_date': '20130513',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist.sxml URL
+        'url': 'http://www.bbc.com/sport/0/football/33653409',
+        'info_dict': {
+            'id': 'p02xycnp',
+            'ext': 'flv',
+            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
+            'duration': 140,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist URL from weather section
+        'url': 'http://www.bbc.com/weather/features/33601775',
+        'only_matching': True,
+    }, {
+        # custom redirection to www.bbc.com
+        'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+
+    def _extract_from_media_meta(self, media_meta, video_id):
+        # Direct links to media in media metadata (e.g.
+        # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+        # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
+        source_files = media_meta.get('sourceFiles')
+        if source_files:
+            return [{
+                'url': f['url'],
+                'format_id': format_id,
+                'ext': f.get('encoding'),
+                'tbr': float_or_none(f.get('bitrate'), 1000),
+                'filesize': int_or_none(f.get('filesize')),
+            } for format_id, f in source_files.items() if f.get('url')], []
+
+        programme_id = media_meta.get('externalId')
+        if programme_id:
+            return self._download_media_selector(programme_id)
+
+        # Process playlist.sxml as legacy playlist
+        href = media_meta.get('href')
+        if href:
+            playlist = self._download_legacy_playlist_url(href)
+            _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
+            return formats, subtitles
+
+        return [], []
+
     def _real_extract(self, url):
-        list_id = self._match_id(url)
-        webpage = self._download_webpage(url, list_id)
+        playlist_id = self._match_id(url)
 
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'list title')
+        webpage = self._download_webpage(url, playlist_id)
 
-        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
-        if pubdate:
-            pubdate = pubdate.replace('-', '')
-
-        ret = []
-        jsent = []
-
-        # works with bbc.com/news/something-something-123456 articles
-        jsent = map(
-            lambda m: self._parse_json(m, list_id),
-            re.findall(r"data-media-meta='({[^']+})'", webpage)
-        )
-
-        if len(jsent) == 0:
-            # http://www.bbc.com/news/video_and_audio/international
-            # and single-video articles
-            masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
-            if masset:
-                jmasset = self._parse_json(masset, list_id)
-                for key, val in jmasset.get('videos', {}).items():
-                    for skey, sval in val.items():
-                        sval['id'] = skey
-                        jsent.append(sval)
-
-        if len(jsent) == 0:
-            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-            # in http://www.bbc.com/news/video_and_audio/international
-            # prone to breaking if entries have sourceFiles list
-            jsent = map(
-                lambda m: self._parse_json(m, list_id),
-                re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-            )
-
-        if len(jsent) == 0:
-            raise ExtractorError('No video found', expected=True)
-
-        for jent in jsent:
-            programme_id = jent.get('externalId')
-            xml_url = jent.get('href')
-
-            title = jent.get('caption', '')
-            if title == '':
-                title = list_title
-
-            duration = parse_duration(jent.get('duration'))
-            description = list_title
-            if jent.get('caption', '') != '':
-                description += ' - ' + jent.get('caption')
-            thumbnail = None
-            if jent.get('image') is not None:
-                thumbnail = jent['image'].get('href')
-
-            formats = []
-            subtitles = []
-
-            if programme_id:
-                formats, subtitles = self._download_media_selector(programme_id)
-            elif jent.get('sourceFiles') is not None:
-                # mediaselector not used at
-                # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
-                for key, val in jent['sourceFiles'].items():
-                    formats.append({
-                        'ext': val.get('encoding'),
-                        'url': val.get('url'),
-                        'filesize': int(val.get('filesize')),
-                        'format_id': key
-                    })
-            elif xml_url:
-                # Cheap fallback
-                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-                formats, subtitles = self._download_media_selector(programme_id)
-
-            if len(formats) == 0:
-                raise ExtractorError('unsupported json media entry.\n    ' + str(jent) + '\n')
+        timestamp = parse_iso8601(self._search_regex(
+            [r'"datePublished":\s*"([^"]+)',
+             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
+            webpage, 'date', default=None))
 
+        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
+        playlist = self._search_regex(
+            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
+            webpage, 'playlist', default=None)
+        if playlist:
+            programme_id, title, description, duration, formats, subtitles = \
+                self._process_legacy_playlist_url(playlist, playlist_id)
             self._sort_formats(formats)
-
-            id = jent.get('id') if programme_id is None else programme_id
-            if id is None:
-                id = 'NA'
-
-            ret.append({
-                'id': id,
-                'uploader': 'BBC News',
-                'upload_date': pubdate,
+            return {
+                'id': programme_id,
                 'title': title,
                 'description': description,
-                'thumbnail': thumbnail,
                 'duration': duration,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        programme_id = self._search_regex(
+            [r'data-video-player-vpid="([\da-z]{8})"',
+             r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+            webpage, 'vpid', default=None)
+        if programme_id:
+            formats, subtitles = self._download_media_selector(programme_id)
+            self._sort_formats(formats)
+            # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
+            digital_data = self._parse_json(
+                self._search_regex(
+                    r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
+                programme_id, fatal=False)
+            page_info = digital_data.get('page', {}).get('pageInfo', {})
+            title = page_info.get('pageName') or self._og_search_title(webpage)
+            description = page_info.get('description') or self._og_search_description(webpage)
+            timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
+            return {
+                'id': programme_id,
+                'title': title,
+                'description': description,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        playlist_title = self._html_search_regex(
+            r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
+        playlist_description = self._og_search_description(webpage)
+
+        # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
+        medias = list(filter(None, map(
+            lambda s: self._parse_json(s, playlist_id, fatal=False),
+            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+
+        if not medias:
+            # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
+            media_asset_page = self._parse_json(
+                self._search_regex(
+                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                playlist_id)
+            medias = []
+            for video in media_asset_page.get('videos', {}).values():
+                medias.extend(video.values())
+
+        entries = []
+        for num, media_meta in enumerate(medias, start=1):
+            formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
+            if not formats:
+                continue
+            self._sort_formats(formats)
+
+            video_id = media_meta.get('externalId')
+            if not video_id:
+                video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)
+
+            title = media_meta.get('caption')
+            if not title:
+                title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)
+
+            duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))
+
+            images = []
+            for image in media_meta.get('images', {}).values():
+                images.extend(image.values())
+            if 'image' in media_meta:
+                images.append(media_meta['image'])
+
+            thumbnails = [{
+                'url': image.get('href'),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            } for image in images]
+
+            entries.append({
+                'id': video_id,
+                'title': title,
+                'thumbnails': thumbnails,
+                'duration': duration,
+                'timestamp': timestamp,
                 'formats': formats,
                 'subtitles': subtitles,
             })
 
-        if len(ret) > 0:
-            return self.playlist_result(ret, list_id, list_title)
-        raise ExtractorError('No video found', expected=True)
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

From cf7e015f250d806916bed1ed2a1dfd8d943c05c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 21:30:34 +0600
Subject: [PATCH 1382/1515] [youtube] Add support for multifeed videos

---
 youtube_dl/extractor/youtube.py | 126 +++++++++++++++++++++++++-------
 1 file changed, 99 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4023a6e50..afe0a781b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,9 +33,11 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    smuggle_url,
     str_to_int,
     unescapeHTML,
     unified_strdate,
+    unsmuggle_url,
     uppercase_escape,
     ISO3166Utils,
 )
@@ -558,6 +560,59 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'format': '135',  # bestvideo
             }
         },
+        {
+            # Multifeed videos (multiple cameras), URL is for Main Camera
+            'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
+            'info_dict': {
+                'id': 'jqWvoWXjCVs',
+                'title': 'teamPGP: Rocket League Noob Stream',
+                'description': 'md5:dc7872fb300e143831327f1bae3af010',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': 'jqWvoWXjCVs',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': '6h8e8xoXJzg',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': 'PUOgX5z9xZw',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': 'teuwxikvS5k',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (zim)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }],
+            'params': {
+                'skip_download': True,
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -889,6 +944,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return formats
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         proto = (
             'http' if self._downloader.params.get('prefer_insecure', False)
             else 'https')
@@ -1005,6 +1062,48 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     '"token" parameter not in video info for unknown reason',
                     video_id=video_id)
 
+        # title
+        if 'title' in video_info:
+            video_title = video_info['title'][0]
+        else:
+            self._downloader.report_warning('Unable to extract video title')
+            video_title = '_'
+
+        # description
+        video_description = get_element_by_id("eow-description", video_webpage)
+        if video_description:
+            video_description = re.sub(r'''(?x)
+                <a\s+
+                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    title="([^"]+)"\s+
+                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    class="yt-uix-redirect-link"\s*>
+                [^<]+
+                </a>
+            ''', r'\1', video_description)
+            video_description = clean_html(video_description)
+        else:
+            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
+            if fd_mobj:
+                video_description = unescapeHTML(fd_mobj.group(1))
+            else:
+                video_description = ''
+
+        if not smuggled_data.get('force_singlefeed', False) and 'multifeed_metadata_list' in video_info:
+            entries = []
+            multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+            for feed in multifeed_metadata_list.split(','):
+                feed_data = compat_parse_qs(feed)
+                entries.append({
+                    '_type': 'url_transparent',
+                    'ie_key': 'Youtube',
+                    'url': smuggle_url(
+                        '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                        {'force_singlefeed': True}),
+                    'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                })
+            return self.playlist_result(entries, video_id, video_title, video_description)
+
         if 'view_count' in video_info:
             view_count = int(video_info['view_count'][0])
         else:
@@ -1030,13 +1129,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
-        # title
-        if 'title' in video_info:
-            video_title = video_info['title'][0]
-        else:
-            self._downloader.report_warning('Unable to extract video title')
-            video_title = '_'
-
         # thumbnail image
         # We try first to get a high quality image:
         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
@@ -1072,26 +1164,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        # description
-        video_description = get_element_by_id("eow-description", video_webpage)
-        if video_description:
-            video_description = re.sub(r'''(?x)
-                <a\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    title="([^"]+)"\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="yt-uix-redirect-link"\s*>
-                [^<]+
-                </a>
-            ''', r'\1', video_description)
-            video_description = clean_html(video_description)
-        else:
-            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
-            if fd_mobj:
-                video_description = unescapeHTML(fd_mobj.group(1))
-            else:
-                video_description = ''
-
         def _extract_count(count_name):
             return str_to_int(self._search_regex(
                 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'

From d8f0a9ecea6aab5b148d06904934c6a504ba3b03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:17:06 +0600
Subject: [PATCH 1383/1515] [youtube] Respect noplaylist for multifeed videos

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index afe0a781b..bcd27408d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1089,7 +1089,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if not smuggled_data.get('force_singlefeed', False) and 'multifeed_metadata_list' in video_info:
+        if (not self._downloader.params.get('noplaylist') and
+                not smuggled_data.get('force_singlefeed', False) and
+                'multifeed_metadata_list' in video_info):
             entries = []
             multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
             for feed in multifeed_metadata_list.split(','):

From 51da40e6218f1dda2fc61650c308194e9b4acbc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:19:54 +0600
Subject: [PATCH 1384/1515] [bbc] PEP8

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2a0901ee4..4b23f82ca 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -423,7 +423,7 @@ class BBCIE(BBCCoUkIE):
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
     # fails with notukerror for some videos
-    #_MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
     _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{

From 7a896817226405a772baa3808d63062d4ad11c94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:32:54 +0600
Subject: [PATCH 1385/1515] [bbc] Skip DASH until supported

---
 youtube_dl/extractor/bbc.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4b23f82ca..66e52641b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -172,6 +172,7 @@ class BBCCoUkIE(InfoExtractor):
         supplier = connection.get('supplier')
         if protocol == 'http':
             href = connection.get('href')
+            transfer_format = connection.get('transferFormat')
             # ASX playlist
             if supplier == 'asx':
                 for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
@@ -179,6 +180,9 @@ class BBCCoUkIE(InfoExtractor):
                         'url': ref,
                         'format_id': 'ref%s_%s' % (i, supplier),
                     })
+            # Skip DASH until supported
+            elif transfer_format == 'dash':
+                pass
             # Direct link
             else:
                 formats.append({

From 5bdec59de15b9bde73a3077a6b9ce517c10b9906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jul 2015 09:51:54 +0600
Subject: [PATCH 1386/1515] [comcarcoff] Add support for singleshots (Closes
 #6366)

---
 youtube_dl/extractor/comcarcoff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 9c25b2223..81f3d7697 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -36,7 +36,7 @@ class ComCarCoffIE(InfoExtractor):
             webpage, 'full data json'))
 
         video_id = full_data['activeVideo']['video']
-        video_data = full_data['videos'][video_id]
+        video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {

From aeb7b41d44313b6bb007b2f1cd0fc8cda84e59d5 Mon Sep 17 00:00:00 2001
From: tippfeler <tippfeler@gmx.de>
Date: Sun, 26 Jul 2015 12:57:06 +0200
Subject: [PATCH 1387/1515] [spiegel] Accept iframe urls

Closes #6370.
---
 youtube_dl/extractor/spiegel.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index b868241d5..5bd3c0087 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -9,7 +9,7 @@ from .spiegeltv import SpiegeltvIE
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed|-iframe)?(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -39,6 +39,9 @@ class SpiegelIE(InfoExtractor):
             'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
             'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
         }
+    }, {
+        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4c6bd5b5b61adfd912e14f8d704fde47628d164e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 26 Jul 2015 14:14:28 +0200
Subject: [PATCH 1388/1515] [youtube] Use 'vp8' and 'vp9' in lowercase (fixes
 #6358)

That's how YouTube reports them in their DASH manifest.
---
 youtube_dl/extractor/youtube.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4023a6e50..0e411bfb6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -281,13 +281,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'VP9'},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
@@ -297,11 +297,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From 2c7ed247967f3563d26879f2206e0f54dfdf9b23 Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Sun, 26 Jul 2015 16:37:51 +0200
Subject: [PATCH 1389/1515] Remove redundant (and wrong) class parameters

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ae813099d..88f9f9070 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -576,11 +576,9 @@ class ContentTooShortError(Exception):
     download is too small for what the server announced first, indicating
     the connection was probably interrupted.
     """
-    # Both in bytes
-    downloaded = None
-    expected = None
 
     def __init__(self, downloaded, expected):
+        # Both in bytes
         self.downloaded = downloaded
         self.expected = expected
 

From cb23bcba294563857561914a19e7d06990c71829 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 27 Jul 2015 04:24:04 -0500
Subject: [PATCH 1390/1515] BBCIE: Fix missing .mp4 formats on news sites

Prior to merge BBCNewsIE from pr #6026 was rewritten into BBCIE in 9afa177. Support was added for non-news
sites (/travel, /sports), however support for the news mediaselector was removed to support these sites.
This removed support for .mp4.

    pr #6026 ( news site, news mediaseletor : http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ )
    format code                                          extension  resolution note
    journalism_nonuk_stream_h264_flv_200_akamai          flv        400x224    h264@ 176k, 1001.77KiB
    journalism_nonuk_stream_h264_flv_400_akamai          flv        400x224    h264@ 512k, 2.85MiB
    journalism_nonuk_stream_h264_flv_med_akamai          flv        640x360    h264@ 800k, 4.43MiB
    journalism_nonuk_stream_h264_flv_hi_akamai           flv        688x384    h264@1500k, 8.34MiB
    journalism_world_stream_h264_http_200_sis_news_http  mp4        400x224    h264@ 176k, 1001.77KiB
    journalism_world_stream_h264_http_400_sis_news_http  mp4        400x224    h264@ 512k, 2.85MiB
    journalism_world_stream_h264_http_hi_sis_news_http   mp4        688x384    h264@1500k, 8.34MiB (best)

    # 9afa177 ( same url, non-news mediaselector: http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ )
    format code                                  extension  resolution note
    journalism_nonuk_stream_h264_flv_lo_akamai   flv        384x216    h264@ 496k, 2.76MiB
    journalism_nonuk_stream_h264_flv_med_akamai  flv        640x360    h264@ 800k, 4.43MiB
    journalism_nonuk_stream_h264_flv_hi_akamai   flv        688x384    h264@1500k, 8.34MiB (best)

This change corrects the above, by trying /mediaselector/5 if /mediaselector/4 fails.
---
 youtube_dl/extractor/bbc.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 66e52641b..0f0ea7cfd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -14,7 +14,6 @@ from ..utils import (
 )
 from ..compat import compat_HTTPError
 
-
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
@@ -271,8 +270,16 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
-        return self._download_media_selector_url(
-            self._MEDIASELECTOR_URL % programme_id, programme_id)
+        try:
+            return self._download_media_selector_url(
+                self._MEDIASELECTOR_URL % programme_id, programme_id)
+        except ExtractorError as e:
+            if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror':
+                 # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/
+                 return self._download_media_selector_url(
+                     self._MEDIASELECTOR_ALT_URL % programme_id, programme_id)
+            else:
+                 raise
 
     def _download_media_selector_url(self, url, programme_id=None):
         try:
@@ -297,7 +304,6 @@ class BBCCoUkIE(InfoExtractor):
                 formats.extend(self._extract_video(media, programme_id))
             elif kind == 'captions':
                 subtitles = self.extract_subtitles(media, programme_id)
-
         return formats, subtitles
 
     def _download_playlist(self, playlist_id):
@@ -426,9 +432,10 @@ class BBCIE(BBCCoUkIE):
     IE_DESC = 'BBC'
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    # fails with notukerror for some videos
-    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+    # fails with notukerror for some videos ( non news sites such as bbc.com/travel )
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # limited selection of formats but may work where the above does not
+    _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{
         # article with multiple videos embedded with data-media-meta containing
@@ -455,7 +462,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
             'timestamp': 1427219242,
@@ -515,7 +522,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
             'timestamp': 1368473503,
@@ -530,7 +537,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
             'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
             'duration': 140,

From 88ed52aec9f2e622188f304d74f2f5568b0caa1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jul 2015 22:05:51 +0600
Subject: [PATCH 1391/1515] [bbc] Add support for direct bbc.co.uk embeds

---
 youtube_dl/extractor/bbc.py     | 34 +++++++++++++++++++++++++++++----
 youtube_dl/extractor/generic.py |  8 --------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 66e52641b..c0433eabd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -450,6 +450,14 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 9,
         'skip': 'Save time',
+    }, {
+        # article with multiple videos embedded with `new SMP()`
+        'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
+        'info_dict': {
+            'id': '3662a707-0af9-3149-963f-47bea720b460',
+            'title': 'BBC Blogs - Adam Curtis - BUGGER',
+        },
+        'playlist_count': 18,
     }, {
         # single video embedded with mediaAssetPage.init()
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -637,12 +645,30 @@ class BBCIE(BBCCoUkIE):
 
         playlist_title = self._html_search_regex(
             r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
-        playlist_description = self._og_search_description(webpage)
+        playlist_description = self._og_search_description(webpage, default=None)
+
+        def extract_all(pattern):
+            return list(filter(None, map(
+                lambda s: self._parse_json(s, playlist_id, fatal=False),
+                re.findall(pattern, webpage))))
+
+        # Multiple video article (e.g.
+        # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]*)?'
+        entries = []
+        for match in extract_all(r'new\s+SMP\(({.+?})\)'):
+            embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
+            if embed_url and re.match(EMBED_URL, embed_url):
+                entries.append(embed_url)
+        entries.extend(re.findall(
+            r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
+        if entries:
+            return self.playlist_result(
+                [self.url_result(entry, 'BBCCoUk') for entry in entries],
+                playlist_id, playlist_title, playlist_description)
 
         # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
-        medias = list(filter(None, map(
-            lambda s: self._parse_json(s, playlist_id, fatal=False),
-            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+        medias = extract_all(r"data-media-meta='({[^']+})'")
 
         if not medias:
             # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6d2efb22e..8cef61c3c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -276,14 +276,6 @@ class GenericIE(InfoExtractor):
                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
             },
         },
-        # BBC iPlayer embeds
-        {
-            'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
-            'info_dict': {
-                'title': 'BBC - Blogs -  Adam Curtis - BUGGER',
-            },
-            'playlist_mincount': 18,
-        },
         # RUTV embed
         {
             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',

From d247a2c8bf2fe4e071cb35a2e2777fe16acecab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Jul 2015 02:06:27 +0600
Subject: [PATCH 1392/1515] [bbc] Fix regex

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c0433eabd..01d07c9c0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -654,7 +654,7 @@ class BBCIE(BBCCoUkIE):
 
         # Multiple video article (e.g.
         # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
-        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]*)?'
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
         entries = []
         for match in extract_all(r'new\s+SMP\(({.+?})\)'):
             embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')

From 289bbb350e11ae1460db515560345088d7c58663 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Jul 2015 11:28:33 +0200
Subject: [PATCH 1393/1515] release 2015.07.28

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 73445137f..657935dc6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -51,6 +51,7 @@
  - **bambuser:channel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **BeatportPro**
  - **Beeg**
@@ -224,6 +225,7 @@
  - **InternetVideoArchive**
  - **IPrima**
  - **iqiyi**: 爱奇艺
+ - **Ir90Tv**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -252,6 +254,7 @@
  - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
+ - **Lecture2Go**
  - **Letv**: 乐视网
  - **LetvPlaylist**
  - **LetvTv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 280afdd7f..fa157cadb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.21'
+__version__ = '2015.07.28'

From f171bc8b59ecf4560dd4076be56570a4f090d519 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:14:06 +0300
Subject: [PATCH 1394/1515] [youtube] save keywords in info jason when
 --write-info-json is used

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0e411bfb6..15e327ec8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,6 +1072,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
+        m = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
+        , video_webpage, re.DOTALL | re.IGNORECASE);
+        video_tags = ", ".join(m)
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
@@ -1259,6 +1262,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'title': video_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
+            'tags' : video_tags,
             'categories': video_categories,
             'subtitles': video_subtitles,
             'automatic_captions': automatic_captions,

From a6f774e9015995393a086273df8db1d7b0c098c4 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:29:13 +0300
Subject: [PATCH 1395/1515] [youtube]: tags key in info jason is now a list

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 15e327ec8..c0fafbfd5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,9 +1072,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        m = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
+        video_tags = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
         , video_webpage, re.DOTALL | re.IGNORECASE);
-        video_tags = ", ".join(m)
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:

From 5316bf7487b608b7c085950ff2fb0444f2c36dc0 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:30:42 +0300
Subject: [PATCH 1396/1515] Documented tags as a possible dict key

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 14b9b4fe2..a227aeb9c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -187,6 +187,7 @@ class InfoExtractor(object):
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
+    tags:           A list of keywords attached to the video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 

From 95d8f7ea12f40a986b541ee17cdf384dbedcfa55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:26:16 +0600
Subject: [PATCH 1397/1515] [fragment] Generalize fragmented media file
 downloader

---
 youtube_dl/downloader/fragment.py | 110 ++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 youtube_dl/downloader/fragment.py

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
new file mode 100644
index 000000000..5f9d6796d
--- /dev/null
+++ b/youtube_dl/downloader/fragment.py
@@ -0,0 +1,110 @@
+from __future__ import division, unicode_literals
+
+import os
+import time
+
+from .common import FileDownloader
+from .http import HttpFD
+from ..utils import (
+    encodeFilename,
+    sanitize_open,
+)
+
+
+class HttpQuietDownloader(HttpFD):
+    def to_screen(self, *args, **kargs):
+        pass
+
+
+class FragmentFD(FileDownloader):
+    """
+    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+    """
+
+    def _prepare_and_start_frag_download(self, ctx):
+        self._prepare_frag_download(ctx)
+        self._start_frag_download(ctx)
+
+    def _prepare_frag_download(self, ctx):
+        self.to_screen('[%s] Total fragments: %d' % (self.FD_NAME, ctx['total_frags']))
+        self.report_destination(ctx['filename'])
+        dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': True,
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
+                'test': self.params.get('test', False),
+            }
+        )
+        tmpfilename = self.temp_name(ctx['filename'])
+        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
+        ctx.update({
+            'dl': dl,
+            'dest_stream': dest_stream,
+            'tmpfilename': tmpfilename,
+        })
+
+    def _start_frag_download(self, ctx):
+        total_frags = ctx['total_frags']
+        # This dict stores the download progress, it's updated by the progress
+        # hook
+        state = {
+            'status': 'downloading',
+            'downloaded_bytes': 0,
+            'frag_index': 0,
+            'frag_count': total_frags,
+            'filename': ctx['filename'],
+            'tmpfilename': ctx['tmpfilename'],
+        }
+        start = time.time()
+        ctx['started'] = start
+
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            frag_total_bytes = s.get('total_bytes', 0)
+            if s['status'] == 'finished':
+                state['downloaded_bytes'] += frag_total_bytes
+                state['frag_index'] += 1
+
+            estimated_size = (
+                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['frag_index'] + 1) * total_frags)
+            time_now = time.time()
+            state['total_bytes_estimate'] = estimated_size
+            state['elapsed'] = time_now - start
+
+            if s['status'] == 'finished':
+                progress = self.calc_percent(state['frag_index'], total_frags)
+            else:
+                frag_downloaded_bytes = s['downloaded_bytes']
+                frag_progress = self.calc_percent(frag_downloaded_bytes,
+                                                  frag_total_bytes)
+                progress = self.calc_percent(state['frag_index'], total_frags)
+                progress += frag_progress / float(total_frags)
+
+                state['eta'] = self.calc_eta(
+                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                state['speed'] = s.get('speed')
+            self._hook_progress(state)
+
+        ctx['dl'].add_progress_hook(frag_progress_hook)
+
+        return start
+
+    def _finish_frag_download(self, ctx):
+        ctx['dest_stream'].close()
+        elapsed = time.time() - ctx['started']
+        self.try_rename(ctx['tmpfilename'], ctx['filename'])
+        fsize = os.path.getsize(encodeFilename(ctx['filename']))
+
+        self._hook_progress({
+            'downloaded_bytes': fsize,
+            'total_bytes': fsize,
+            'filename': ctx['filename'],
+            'status': 'finished',
+            'elapsed': elapsed,
+        })

From ab81ef8fa7d0f9e814890024140c7ed9587d3151 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:27:50 +0600
Subject: [PATCH 1398/1515] [f4m] Implement f4m fd in terms of fragment fd

---
 youtube_dl/downloader/f4m.py | 98 +++++++-----------------------------
 1 file changed, 17 insertions(+), 81 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b1a858c45..275564b59 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -7,8 +7,7 @@ import os
 import time
 import xml.etree.ElementTree as etree
 
-from .common import FileDownloader
-from .http import HttpFD
+from .fragment import FragmentFD
 from ..compat import (
     compat_urlparse,
     compat_urllib_error,
@@ -16,8 +15,6 @@ from ..compat import (
 from ..utils import (
     struct_pack,
     struct_unpack,
-    encodeFilename,
-    sanitize_open,
     xpath_text,
 )
 
@@ -226,16 +223,13 @@ def _add_ns(prop):
     return '{http://ns.adobe.com/f4m/1.0}%s' % prop
 
 
-class HttpQuietDownloader(HttpFD):
-    def to_screen(self, *args, **kargs):
-        pass
-
-
-class F4mFD(FileDownloader):
+class F4mFD(FragmentFD):
     """
     A downloader for f4m manifests or AdobeHDS.
     """
 
+    FD_NAME = 'f4m'
+
     def _get_unencrypted_media(self, doc):
         media = doc.findall(_add_ns('media'))
         if not media:
@@ -288,7 +282,7 @@ class F4mFD(FileDownloader):
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
-        self.to_screen('[download] Downloading f4m manifest')
+        self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
         manifest = self.ydl.urlopen(man_url).read()
 
         doc = etree.fromstring(manifest)
@@ -320,67 +314,20 @@ class F4mFD(FileDownloader):
         # For some akamai manifests we'll need to add a query to the fragment url
         akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
-        self.report_destination(filename)
-        http_dl = HttpQuietDownloader(
-            self.ydl,
-            {
-                'continuedl': True,
-                'quiet': True,
-                'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
-                'test': self.params.get('test', False),
-            }
-        )
-        tmpfilename = self.temp_name(filename)
-        (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+        ctx = {
+            'filename': filename,
+            'total_frags': total_frags,
+        }
+
+        self._prepare_frag_download(ctx)
+
+        dest_stream = ctx['dest_stream']
 
         write_flv_header(dest_stream)
         if not live:
             write_metadata_tag(dest_stream, metadata)
 
-        # This dict stores the download progress, it's updated by the progress
-        # hook
-        state = {
-            'status': 'downloading',
-            'downloaded_bytes': 0,
-            'frag_index': 0,
-            'frag_count': total_frags,
-            'filename': filename,
-            'tmpfilename': tmpfilename,
-        }
-        start = time.time()
-
-        def frag_progress_hook(s):
-            if s['status'] not in ('downloading', 'finished'):
-                return
-
-            frag_total_bytes = s.get('total_bytes', 0)
-            if s['status'] == 'finished':
-                state['downloaded_bytes'] += frag_total_bytes
-                state['frag_index'] += 1
-
-            estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes) /
-                (state['frag_index'] + 1) * total_frags)
-            time_now = time.time()
-            state['total_bytes_estimate'] = estimated_size
-            state['elapsed'] = time_now - start
-
-            if s['status'] == 'finished':
-                progress = self.calc_percent(state['frag_index'], total_frags)
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                frag_progress = self.calc_percent(frag_downloaded_bytes,
-                                                  frag_total_bytes)
-                progress = self.calc_percent(state['frag_index'], total_frags)
-                progress += frag_progress / float(total_frags)
-
-                state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
-                state['speed'] = s.get('speed')
-            self._hook_progress(state)
-
-        http_dl.add_progress_hook(frag_progress_hook)
+        self._start_frag_download(ctx)
 
         frags_filenames = []
         while fragments_list:
@@ -391,9 +338,9 @@ class F4mFD(FileDownloader):
                 url += '?' + akamai_pv.strip(';')
             if info_dict.get('extra_param_to_segment_url'):
                 url += info_dict.get('extra_param_to_segment_url')
-            frag_filename = '%s-%s' % (tmpfilename, name)
+            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = http_dl.download(frag_filename, {'url': url})
+                success = ctx['dl'].download(frag_filename, {'url': url})
                 if not success:
                     return False
                 with open(frag_filename, 'rb') as down:
@@ -425,20 +372,9 @@ class F4mFD(FileDownloader):
                     msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
                     self.report_warning(msg)
 
-        dest_stream.close()
+        self._finish_frag_download(ctx)
 
-        elapsed = time.time() - start
-        self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
             os.remove(frag_file)
 
-        fsize = os.path.getsize(encodeFilename(filename))
-        self._hook_progress({
-            'downloaded_bytes': fsize,
-            'total_bytes': fsize,
-            'filename': filename,
-            'status': 'finished',
-            'elapsed': elapsed,
-        })
-
         return True

From f9a5affad968c6629c022f489dc3c1561a4c57de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:28:30 +0600
Subject: [PATCH 1399/1515] [hls] Implement hlsnative fd in terms of fragment
 fd

---
 youtube_dl/downloader/hls.py | 81 +++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8be4f4249..60dca0ab1 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -4,12 +4,11 @@ import os
 import re
 import subprocess
 
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_request,
-)
+from .fragment import FragmentFD
+
+from ..compat import compat_urlparse
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
     encodeArgument,
     encodeFilename,
@@ -51,54 +50,50 @@ class HlsFD(FileDownloader):
             return False
 
 
-class NativeHlsFD(FileDownloader):
+class NativeHlsFD(FragmentFD):
     """ A more limited implementation that does not require ffmpeg """
 
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
+    FD_NAME = 'hlsnative'
 
-        self.to_screen(
-            '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id'])
-        data = self.ydl.urlopen(url).read()
-        s = data.decode('utf-8', 'ignore')
-        segment_urls = []
+    def real_download(self, filename, info_dict):
+        man_url = info_dict['url']
+        self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
+        manifest = self.ydl.urlopen(man_url).read()
+
+        s = manifest.decode('utf-8', 'ignore')
+        fragment_urls = []
         for line in s.splitlines():
             line = line.strip()
             if line and not line.startswith('#'):
                 segment_url = (
                     line
                     if re.match(r'^https?://', line)
-                    else compat_urlparse.urljoin(url, line))
-                segment_urls.append(segment_url)
-
-        is_test = self.params.get('test', False)
-        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
-        byte_counter = 0
-        with open(tmpfilename, 'wb') as outf:
-            for i, segurl in enumerate(segment_urls):
-                self.to_screen(
-                    '[hlsnative] %s: Downloading segment %d / %d' %
-                    (info_dict['id'], i + 1, len(segment_urls)))
-                seg_req = compat_urllib_request.Request(segurl)
-                if remaining_bytes is not None:
-                    seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
-
-                segment = self.ydl.urlopen(seg_req).read()
-                if remaining_bytes is not None:
-                    segment = segment[:remaining_bytes]
-                    remaining_bytes -= len(segment)
-                outf.write(segment)
-                byte_counter += len(segment)
-                if remaining_bytes is not None and remaining_bytes <= 0:
+                    else compat_urlparse.urljoin(man_url, line))
+                fragment_urls.append(segment_url)
+                # We only download the first fragment during the test
+                if self.params.get('test', False):
                     break
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': byte_counter,
+        ctx = {
             'filename': filename,
-            'status': 'finished',
-        })
-        self.try_rename(tmpfilename, filename)
+            'total_frags': len(fragment_urls),
+        }
+
+        self._prepare_and_start_frag_download(ctx)
+
+        frags_filenames = []
+        for i, frag_url in enumerate(fragment_urls):
+            frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+            success = ctx['dl'].download(frag_filename, {'url': frag_url})
+            if not success:
+                return False
+            with open(frag_filename, 'rb') as down:
+                ctx['dest_stream'].write(down.read())
+            frags_filenames.append(frag_filename)
+
+        self._finish_frag_download(ctx)
+
+        for frag_file in frags_filenames:
+            os.remove(frag_file)
+
         return True

From 864f24bd2c0cf9bde034812a2049c3750c1bb05c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 03:43:03 +0600
Subject: [PATCH 1400/1515] [extractor/common] Add _meta_regex and clarify tags
 field

---
 youtube_dl/extractor/common.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a227aeb9c..d54866d1f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -181,13 +181,13 @@ class InfoExtractor(object):
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
                     ["Sports", "Berlin"]
+    tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
-    tags:           A list of keywords attached to the video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -631,6 +631,12 @@ class InfoExtractor(object):
             template % (content_re, property_re),
         ]
 
+    @staticmethod
+    def _meta_regex(prop):
+        return r'''(?isx)<meta
+                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
+
     def _og_search_property(self, prop, html, name=None, **kargs):
         if name is None:
             name = 'OpenGraph %s' % prop
@@ -661,9 +667,7 @@ class InfoExtractor(object):
         if display_name is None:
             display_name = name
         return self._html_search_regex(
-            r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
-                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
+            self._meta_regex(name),
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):

From 000b6b5ae5cc214906effe4ac5b78b579bc7db70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 03:43:32 +0600
Subject: [PATCH 1401/1515] [youtube] Improve tags extraction and add test

---
 youtube_dl/extractor/youtube.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c0fafbfd5..4c449fd74 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -329,6 +329,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20121002',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
+                'tags': ['youtube-dl'],
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
@@ -343,7 +344,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
-                'description': 'md5:fea86fda2d5a5784273df5c7cc994d9f',
+                'description': 'md5:782e8651347686cba06e58f71ab51773',
+                'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
+                         'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
+                         'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
             }
@@ -1072,8 +1076,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        video_tags = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
-        , video_webpage, re.DOTALL | re.IGNORECASE);
+        video_tags = [
+            unescapeHTML(m.group('content'))
+            for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
+
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
@@ -1261,8 +1267,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'title': video_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
-            'tags' : video_tags,
             'categories': video_categories,
+            'tags': video_tags,
             'subtitles': video_subtitles,
             'automatic_captions': automatic_captions,
             'duration': video_duration,

From 5e1eddb939a4fbc2a5ef10111c8141c842cf01d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 21:18:16 +0600
Subject: [PATCH 1402/1515] [youtube] Show info message for multifeed videos
 according to noplaylist option

---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bcd27408d..8a5ef2e70 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1089,22 +1089,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if (not self._downloader.params.get('noplaylist') and
-                not smuggled_data.get('force_singlefeed', False) and
-                'multifeed_metadata_list' in video_info):
-            entries = []
-            multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
-            for feed in multifeed_metadata_list.split(','):
-                feed_data = compat_parse_qs(feed)
-                entries.append({
-                    '_type': 'url_transparent',
-                    'ie_key': 'Youtube',
-                    'url': smuggle_url(
-                        '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
-                        {'force_singlefeed': True}),
-                    'title': '%s (%s)' % (video_title, feed_data['title'][0]),
-                })
-            return self.playlist_result(entries, video_id, video_title, video_description)
+        if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
+            if not self._downloader.params.get('noplaylist'):
+                entries = []
+                feed_ids = []
+                multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+                for feed in multifeed_metadata_list.split(','):
+                    feed_data = compat_parse_qs(feed)
+                    entries.append({
+                        '_type': 'url_transparent',
+                        'ie_key': 'Youtube',
+                        'url': smuggle_url(
+                            '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                            {'force_singlefeed': True}),
+                        'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                    })
+                    feed_ids.append(feed_data['id'][0])
+                self.to_screen(
+                    'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
+                    % (', '.join(feed_ids), video_id))
+                return self.playlist_result(entries, video_id, video_title, video_description)
+            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
         if 'view_count' in video_info:
             view_count = int(video_info['view_count'][0])

From 2711e41bcdaf8e8234dd32e5df6fb76f23179f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Jul 2015 18:47:20 +0200
Subject: [PATCH 1403/1515] Credit slangangular for the sportschau extractor
 (#6199)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e75e9885d..aa6b88cc0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -135,3 +135,4 @@ Bernhard Minks
 sceext
 Zach Bruggeman
 Tjark Saul
+slangangular

From d12a1a47d5ff0833a1ecd7fe47f4ffa67a9b4a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jul 2015 00:55:06 +0600
Subject: [PATCH 1404/1515] [bbc] Improve work with mediaselection URLs

---
 youtube_dl/extractor/bbc.py | 50 ++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 3d9366644..9a1b6e3dc 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -14,12 +14,15 @@ from ..utils import (
 )
 from ..compat import compat_HTTPError
 
+
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+    _MEDIASELECTOR_URLS = [
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
+    ]
 
     _TESTS = [
         {
@@ -161,6 +164,10 @@ class BBCCoUkIE(InfoExtractor):
         }
     ]
 
+    class MediaSelectionError(Exception):
+        def __init__(self, id):
+            self.id = id
+
     def _extract_asx_playlist(self, connection, programme_id):
         asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
         return [ref.get('href') for ref in asx.findall('./Entry/ref')]
@@ -211,8 +218,7 @@ class BBCCoUkIE(InfoExtractor):
     def _extract_medias(self, media_selection):
         error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
         if error is not None:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
+            raise BBCCoUkIE.MediaSelectionError(error.get('id'))
         return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
 
     def _extract_connections(self, media):
@@ -269,17 +275,23 @@ class BBCCoUkIE(InfoExtractor):
             ]
         return subtitles
 
+    def _raise_extractor_error(self, media_selection_error):
+        raise ExtractorError(
+            '%s returned error: %s' % (self.IE_NAME, media_selection_error.id),
+            expected=True)
+
     def _download_media_selector(self, programme_id):
-        try:
-            return self._download_media_selector_url(
-                self._MEDIASELECTOR_URL % programme_id, programme_id)
-        except ExtractorError as e:
-            if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror':
-                 # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/
-                 return self._download_media_selector_url(
-                     self._MEDIASELECTOR_ALT_URL % programme_id, programme_id)
-            else:
-                 raise
+        last_exception = None
+        for mediaselector_url in self._MEDIASELECTOR_URLS:
+            try:
+                return self._download_media_selector_url(
+                    mediaselector_url % programme_id, programme_id)
+            except BBCCoUkIE.MediaSelectionError as e:
+                if e.id == 'notukerror':
+                    last_exception = e
+                    continue
+                self._raise_extractor_error(e)
+        self._raise_extractor_error(last_exception)
 
     def _download_media_selector_url(self, url, programme_id=None):
         try:
@@ -432,10 +444,14 @@ class BBCIE(BBCCoUkIE):
     IE_DESC = 'BBC'
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    # fails with notukerror for some videos ( non news sites such as bbc.com/travel )
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-    # limited selection of formats but may work where the above does not
-    _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+    _MEDIASELECTOR_URLS = [
+        # Provides more formats, namely direct mp4 links, but fails on some videos with
+        # notukerror for non UK (?) users (e.g.
+        # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s',
+        # Provides fewer formats, but works everywhere for everybody (hopefully)
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s',
+    ]
 
     _TESTS = [{
         # article with multiple videos embedded with data-media-meta containing

From 799207e838e0404aaa5cb6658e41bef108aced16 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 29 Jul 2015 23:20:37 +0100
Subject: [PATCH 1405/1515] [viewster] extract the api auth token

Closes #6406.
---
 youtube_dl/compat.py             | 6 ++++++
 youtube_dl/extractor/common.py   | 8 ++++++++
 youtube_dl/extractor/viewster.py | 5 ++++-
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0c57c7aeb..e4b9286c0 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -42,6 +42,11 @@ try:
 except ImportError:  # Python 2
     import cookielib as compat_cookiejar
 
+try:
+    import http.cookies as compat_cookies
+except ImportError:  # Python 2
+    import Cookie as compat_cookies
+
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
@@ -436,6 +441,7 @@ __all__ = [
     'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
+    'compat_cookies',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d54866d1f..dc5080504 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -14,10 +14,12 @@ import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
+    compat_cookies,
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
+    compat_urllib_request,
     compat_urlparse,
     compat_str,
 )
@@ -1074,6 +1076,12 @@ class InfoExtractor(object):
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
+    def _get_cookies(self, url):
+        """ Return a compat_cookies.SimpleCookie with the cookies for the url """
+        req = compat_urllib_request.Request(url)
+        self._downloader.cookiejar.add_cookie_header(req)
+        return compat_cookies.SimpleCookie(req.get_header('Cookie'))
+
     def get_testcases(self, include_onlymatching=False):
         t = getattr(self, '_TEST', None)
         if t:
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 6ef36290b..393b63618 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -62,7 +62,6 @@ class ViewsterIE(InfoExtractor):
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
-    _AUTH_TOKEN = '/YqhSYsx8EaU9Bsta3ojlA=='
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
         request = compat_urllib_request.Request(url)
@@ -72,6 +71,10 @@ class ViewsterIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        # Get 'api_token' cookie
+        self._request_webpage(url, video_id)
+        cookies = self._get_cookies(url)
+        self._AUTH_TOKEN = compat_urllib_parse.unquote(cookies['api_token'].value)
 
         info = self._download_json(
             'https://public-api.viewster.com/search/%s' % video_id,

From 1f04873517e41d2f54fc4e65c46c08ba85b23010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 30 Jul 2015 19:12:37 +0200
Subject: [PATCH 1406/1515] [viewster] Use 'compat_urllib_parse_unquote'

---
 youtube_dl/extractor/viewster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 393b63618..65324d903 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from ..utils import (
     determine_ext,
@@ -74,7 +75,7 @@ class ViewsterIE(InfoExtractor):
         # Get 'api_token' cookie
         self._request_webpage(url, video_id)
         cookies = self._get_cookies(url)
-        self._AUTH_TOKEN = compat_urllib_parse.unquote(cookies['api_token'].value)
+        self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 
         info = self._download_json(
             'https://public-api.viewster.com/search/%s' % video_id,

From 80fb6d4aa47154a1e963b28a17a065dc40a436b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Jul 2015 00:54:26 +0600
Subject: [PATCH 1407/1515] [soundcloud:user] Rework extractor (Closes #6399)

---
 youtube_dl/extractor/soundcloud.py | 131 ++++++++++++++++++++++-------
 1 file changed, 101 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 0a6c9fe72..f1307dc83 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!sets/|(?:likes|tracks)/?(?:$|[?#]))
+                            (?!(?:tracks|sets|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -293,60 +293,131 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|sets|reposts|likes|spotlight)/?)?(\?.*)?$'
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
-        'url': 'https://soundcloud.com/the-concept-band',
+        'url': 'https://soundcloud.com/the-akashic-chronicler',
         'info_dict': {
-            'id': '9615865',
-            'title': 'The Royal Concept',
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (All)',
         },
-        'playlist_mincount': 12
-    }, {
-        'url': 'https://soundcloud.com/the-concept-band/likes',
-        'info_dict': {
-            'id': '9615865',
-            'title': 'The Royal Concept',
-        },
-        'playlist_mincount': 1,
+        'playlist_mincount': 112,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
-        'only_matching': True,
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Tracks)',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Playlists)',
+        },
+        'playlist_mincount': 3,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Reposts)',
+        },
+        'playlist_mincount': 9,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Likes)',
+        },
+        'playlist_mincount': 333,
+    }, {
+        'url': 'https://soundcloud.com/grynpyret/spotlight',
+        'info_dict': {
+            'id': '7098329',
+            'title': 'Grynpyret (Spotlight)',
+        },
+        'playlist_mincount': 1,
     }]
 
+    _API_BASE = 'https://api.soundcloud.com'
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
+
+    _BASE_URL_MAP = {
+        'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
+        'tracks': '%s/users/%%s/tracks' % _API_BASE,
+        'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
+        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
+        'likes': '%s/users/%%s/likes' % _API_V2_BASE,
+        'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
+    }
+
+    _TITLE_MAP = {
+        'all': 'All',
+        'tracks': 'Tracks',
+        'sets': 'Playlists',
+        'reposts': 'Reposts',
+        'likes': 'Likes',
+        'spotlight': 'Spotlight',
+    }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group('user')
-        resource = mobj.group('rsrc')
-        if resource is None:
-            resource = 'tracks'
-        elif resource == 'likes':
-            resource = 'favorites'
 
         url = 'http://soundcloud.com/%s/' % uploader
         resolv_url = self._resolv_url(url)
         user = self._download_json(
             resolv_url, uploader, 'Downloading user info')
-        base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource)
+
+        resource = mobj.group('rsrc') or 'all'
+        base_url = self._BASE_URL_MAP[resource] % user['id']
+
+        next_href = None
 
         entries = []
         for i in itertools.count():
-            data = compat_urllib_parse.urlencode({
-                'offset': i * 50,
-                'limit': 50,
-                'client_id': self._CLIENT_ID,
-            })
-            new_entries = self._download_json(
-                base_url + data, uploader, 'Downloading track page %s' % (i + 1))
-            if len(new_entries) == 0:
+            if not next_href:
+                data = compat_urllib_parse.urlencode({
+                    'offset': i * 50,
+                    'limit': 50,
+                    'client_id': self._CLIENT_ID,
+                    'linked_partitioning': '1',
+                    'representation': 'speedy',
+                })
+                next_href = base_url + '?' + data
+
+            response = self._download_json(
+                next_href, uploader, 'Downloading track page %s' % (i + 1))
+
+            collection = response['collection']
+
+            if not collection:
                 self.to_screen('%s: End page received' % uploader)
                 break
-            entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries)
+
+            def resolve_permalink_url(candidates):
+                for cand in candidates:
+                    if isinstance(cand, dict):
+                        permalink_url = cand.get('permalink_url')
+                        if permalink_url and permalink_url.startswith('http'):
+                            return permalink_url
+
+            for e in collection:
+                permalink_url = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
+                if permalink_url:
+                    entries.append(self.url_result(permalink_url))
+
+            if 'next_href' in response:
+                next_href = response['next_href']
+                if not next_href:
+                    break
+            else:
+                next_href = None
 
         return {
             '_type': 'playlist',
             'id': compat_str(user['id']),
-            'title': user['username'],
+            'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
             'entries': entries,
         }
 

From 09b6468d3027bbc0180cd8b10aa848180cae4005 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 31 Jul 2015 21:28:59 +0100
Subject: [PATCH 1408/1515] [vidme] extract the uploader

---
 youtube_dl/extractor/vidme.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index e0b55078b..575a62f3a 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -50,6 +50,7 @@ class VidmeIE(InfoExtractor):
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
+        uploader = self._html_search_regex('class="video_author_username">([^<]+)', webpage, 'uploader', fatal=False)
 
         return {
             'id': video_id,
@@ -63,4 +64,5 @@ class VidmeIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
+            'uploader': uploader,
         }

From 16a089780e87d57b5295c4dfb8da1217ef6eda70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 03:50:03 +0600
Subject: [PATCH 1409/1515] [soundcloud] Clean up _VALID_URLs

---
 youtube_dl/extractor/soundcloud.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index f1307dc83..6ce86cbcd 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!(?:tracks|sets|reposts|likes|spotlight)/?(?:$|[?#]))
+                            (?!(?:tracks|sets(?:/[^/?#]+)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -293,7 +293,15 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|sets|reposts|likes|spotlight)/?)?(\?.*)?$'
+    _VALID_URL = r'''(?x)
+                        https?://
+                            (?:(?:www|m)\.)?soundcloud\.com/
+                            (?P<user>[^/]+)
+                            (?:/
+                                (?P<rsrc>tracks|sets|reposts|likes|spotlight)
+                            )?
+                            /?(?:[?#].*)?$
+                    '''
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-akashic-chronicler',

From 3b58d94f71b0e7b3ace6f5965f7335aa95fd0c1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 04:03:52 +0600
Subject: [PATCH 1410/1515] [vidme] Make uploader optional and add test

---
 youtube_dl/extractor/vidme.py | 37 ++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 575a62f3a..157bb74fe 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -22,6 +22,27 @@ class VidmeIE(InfoExtractor):
             'timestamp': 1406313244,
             'upload_date': '20140725',
             'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+    }, {
+        # tests uploader field
+        'url': 'https://vid.me/4Iib',
+        'info_dict': {
+            'id': '4Iib',
+            'ext': 'mp4',
+            'title': 'The Carver',
+            'description': 'md5:e9c24870018ae8113be936645b93ba3c',
+            'duration': 97.859999999999999,
+            'timestamp': 1433203629,
+            'upload_date': '20150602',
+            'uploader': 'Thomas',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
@@ -40,17 +61,23 @@ class VidmeIE(InfoExtractor):
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage, default='')
         thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = int_or_none(self._og_search_property('updated_time', webpage, fatal=False))
-        width = int_or_none(self._og_search_property('video:width', webpage, fatal=False))
-        height = int_or_none(self._og_search_property('video:height', webpage, fatal=False))
+        timestamp = int_or_none(self._og_search_property(
+            'updated_time', webpage, fatal=False))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, fatal=False))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, fatal=False))
         duration = float_or_none(self._html_search_regex(
             r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
         view_count = str_to_int(self._html_search_regex(
-            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
+            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?',
+            webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
-        uploader = self._html_search_regex('class="video_author_username">([^<]+)', webpage, 'uploader', fatal=False)
+        uploader = self._html_search_regex(
+            'class="video_author_username"[^>]*>([^<]+)',
+            webpage, 'uploader', default=None)
 
         return {
             'id': video_id,

From 30a453884e6a4d228ced4cdddcfa8e79a768755a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 31 Jul 2015 14:41:30 +0100
Subject: [PATCH 1411/1515] [viewster] use head request to extract api token

Closes #6419.
---
 youtube_dl/extractor/viewster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 65324d903..cda02ba24 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -11,6 +11,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     parse_iso8601,
+    HEADRequest,
 )
 
 
@@ -73,7 +74,7 @@ class ViewsterIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # Get 'api_token' cookie
-        self._request_webpage(url, video_id)
+        self._request_webpage(HEADRequest(url), video_id)
         cookies = self._get_cookies(url)
         self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 

From 44cae2fb2e7b8cffcdfdd57b1224115345009cc5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 30 Jul 2015 23:55:57 +0100
Subject: [PATCH 1412/1515] [gdcvault] Add support for audio extraction (fixes
 #5784)

Closes #6412.
---
 youtube_dl/extractor/gdcvault.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 43f916412..c3017d451 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -7,7 +7,10 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
-from ..utils import remove_end
+from ..utils import (
+    remove_end,
+    HEADRequest,
+)
 
 
 class GDCVaultIE(InfoExtractor):
@@ -73,10 +76,20 @@ class GDCVaultIE(InfoExtractor):
         return video_formats
 
     def _parse_flv(self, xml_description):
-        video_formats = []
+        formats = []
         akamai_url = xml_description.find('./metadata/akamaiHost').text
+        audios = xml_description.find('./metadata/audios')
+        if audios is not None:
+            for audio in audios:
+                formats.append({
+                    'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                    'play_path': remove_end(audio.get('url'), '.flv'),
+                    'ext': 'flv',
+                    'vcodec': 'none',
+                    'format_id': audio.get('code'),
+                })
         slide_video_path = xml_description.find('./metadata/slideVideo').text
-        video_formats.append({
+        formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(slide_video_path, '.flv'),
             'ext': 'flv',
@@ -86,7 +99,7 @@ class GDCVaultIE(InfoExtractor):
             'format_id': 'slides',
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
-        video_formats.append({
+        formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(speaker_video_path, '.flv'),
             'ext': 'flv',
@@ -95,7 +108,7 @@ class GDCVaultIE(InfoExtractor):
             'preference': -1,
             'format_id': 'speaker',
         })
-        return video_formats
+        return formats
 
     def _login(self, webpage_url, display_id):
         (username, password) = self._get_login_info()
@@ -133,16 +146,18 @@ class GDCVaultIE(InfoExtractor):
             r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
             start_page, 'url', default=None)
         if direct_url:
-            video_url = 'http://www.gdcvault.com/' + direct_url
             title = self._html_search_regex(
                 r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>',
                 start_page, 'title')
+            video_url = 'http://www.gdcvault.com' + direct_url
+            # resolve the url so that we can detect the correct extension
+            head = self._request_webpage(HEADRequest(video_url), video_id)
+            video_url = head.geturl()
 
             return {
                 'id': video_id,
                 'display_id': display_id,
                 'url': video_url,
-                'ext': 'flv',
                 'title': title,
             }
 

From 525a87f58ee9c4ee91b5b0384184dabf6d87eef3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 1 Aug 2015 11:40:34 +0200
Subject: [PATCH 1413/1515] [gdcvault] Fix typo: xml_decription_url ->
 xml_description_url

---
 youtube_dl/extractor/gdcvault.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index c3017d451..a6834db43 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -183,8 +183,8 @@ class GDCVaultIE(InfoExtractor):
             # Fallback to the older format
             xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
 
-        xml_decription_url = xml_root + 'xml/' + xml_name
-        xml_description = self._download_xml(xml_decription_url, display_id)
+        xml_description_url = xml_root + 'xml/' + xml_name
+        xml_description = self._download_xml(xml_description_url, display_id)
 
         video_title = xml_description.find('./metadata/title').text
         video_formats = self._parse_mp4(xml_description)

From ee114368ad0bb9822449295910263a99f9de4e1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 20:22:13 +0600
Subject: [PATCH 1414/1515] [utils] Make value optional for find_xpath_attr

This allows selecting particular attributes by name but without specifying the value and similar to xpath syntax `[@attrib]`
---
 test/test_utils.py  |  9 +++++++++
 youtube_dl/utils.py | 13 ++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 65692a9fb..a759b2da9 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -235,12 +235,21 @@ class TestUtil(unittest.TestCase):
             <node x="a"/>
             <node x="a" y="c" />
             <node x="b" y="d" />
+            <node x="" />
         </root>'''
         doc = xml.etree.ElementTree.fromstring(testxml)
 
+        self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'n'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'n', 'v'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x'), doc[1])
         self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'b'), doc[3])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'y'), doc[2])
         self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'd'), doc[3])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x', ''), doc[4])
 
     def test_xpath_with_ns(self):
         testxml = '''<root xmlns:media="http://example.com/">
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 88f9f9070..78dc2b449 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -139,21 +139,24 @@ def write_json_file(obj, fn):
 
 
 if sys.version_info >= (2, 7):
-    def find_xpath_attr(node, xpath, key, val):
+    def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z-]+$', key)
-        assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
-        expr = xpath + "[@%s='%s']" % (key, val)
+        if val:
+            assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
+        expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:
-    def find_xpath_attr(node, xpath, key, val):
+    def find_xpath_attr(node, xpath, key, val=None):
         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
         # .//node does not match if a node is a direct child of . !
         if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
 
         for f in node.findall(xpath):
-            if f.attrib.get(key) == val:
+            if key not in f.attrib:
+                continue
+            if val is None or f.attrib.get(key) == val:
                 return f
         return None
 

From 3f125c8c70e8109bc90d4446b40740133e343b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 21:43:33 +0600
Subject: [PATCH 1415/1515] [nbcnews] Extend _VALID_URL

---
 youtube_dl/extractor/nbc.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index dc2091be0..ccdbfb6c9 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -124,7 +124,7 @@ class NBCSportsIE(InfoExtractor):
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
-        (?:feature|nightly-news)/[^/]+/(?P<title>.+))
+        (?:watch|feature|nightly-news)/[^/]+/(?P<title>.+))
         '''
 
     _TESTS = [
@@ -169,6 +169,10 @@ class NBCNewsIE(InfoExtractor):
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 55eae65b39d754d699ad9de3f9c99fcdf62e0176 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Aug 2015 00:42:23 +0800
Subject: [PATCH 1416/1515] Credit @cyb3r for the ir90tv extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index aa6b88cc0..d16d34272 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -136,3 +136,4 @@ sceext
 Zach Bruggeman
 Tjark Saul
 slangangular
+Behrouz Abbasi

From a107193e4b7a3d5414dd7422263c34ac0e309ec4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:13:21 +0600
Subject: [PATCH 1417/1515] [extractor/common] Extract f4m and m3u8 formats,
 subtitles and info

---
 youtube_dl/extractor/common.py | 200 ++++++++++++++++++++++++---------
 1 file changed, 149 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..f9578b838 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -18,6 +18,7 @@ from ..compat import (
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
@@ -37,6 +38,7 @@ from ..utils import (
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
+    url_basename,
 )
 
 
@@ -978,69 +980,165 @@ class InfoExtractor(object):
         self._sort_formats(formats)
         return formats
 
-    # TODO: improve extraction
-    def _extract_smil_formats(self, smil_url, video_id, fatal=True):
-        smil = self._download_xml(
-            smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file', fatal=fatal)
+    @staticmethod
+    def _xpath_ns(path, namespace=None):
+        if not namespace:
+            return path
+        out = []
+        for c in path.split('/'):
+            if not c or c == '.':
+                out.append(c)
+            else:
+                out.append('{%s}%s' % (namespace, c))
+        return '/'.join(out)
+
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+
         if smil is False:
             assert not fatal
             return []
 
-        base = smil.find('./head/meta').get('base')
+        namespace = self._search_regex(
+            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+
+        return self._parse_smil_formats(
+            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
+
+    def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+        if smil is False:
+            return {}
+        return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
+
+    def _download_smil(self, smil_url, video_id, fatal=True):
+        return self._download_xml(
+            smil_url, video_id, 'Downloading SMIL file',
+            'Unable to download SMIL file', fatal=fatal)
+
+    def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
+        namespace = self._search_regex(
+            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+
+        formats = self._parse_smil_formats(
+            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
+        subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
+
+        video_id = os.path.splitext(url_basename(smil_url))[0]
+        title = None
+        description = None
+        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
+            name = meta.attrib.get('name')
+            content = meta.attrib.get('content')
+            if not name or not content:
+                continue
+            if not title and name == 'title':
+                title = content
+            elif not description and name in ('description', 'abstract'):
+                description = content
+
+        return {
+            'id': video_id,
+            'title': title or video_id,
+            'description': description,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
+        base = smil_url
+        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
+            b = meta.get('base') or meta.get('httpBase')
+            if b:
+                base = b
+                break
 
         formats = []
         rtmp_count = 0
-        if smil.findall('./body/seq/video'):
-            video = smil.findall('./body/seq/video')[0]
-            fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
-            formats.extend(fmts)
-        else:
-            for video in smil.findall('./body/switch/video'):
-                fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
-                formats.extend(fmts)
+        http_count = 0
+
+        videos = smil.findall(self._xpath_ns('.//video', namespace))
+        for video in videos:
+            src = video.get('src')
+            if not src:
+                continue
+
+            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            filesize = int_or_none(video.get('size') or video.get('fileSize'))
+            width = int_or_none(video.get('width'))
+            height = int_or_none(video.get('height'))
+            proto = video.get('proto')
+            ext = video.get('ext')
+            src_ext = determine_ext(src)
+            streamer = video.get('streamer') or base
+
+            if proto == 'rtmp' or streamer.startswith('rtmp'):
+                rtmp_count += 1
+                formats.append({
+                    'url': streamer,
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                    'tbr': bitrate,
+                    'filesize': filesize,
+                    'width': width,
+                    'height': height,
+                })
+                continue
+
+            src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+
+            if proto == 'm3u8' or src_ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls'))
+                continue
+
+            if src_ext == 'f4m':
+                f4m_url = src_url
+                if not f4m_params:
+                    f4m_params = {
+                        'hdcore': '3.2.0',
+                        'plugin': 'flowplayer-3.2.0.1',
+                    }
+                f4m_url += '&' if '?' in f4m_url else '?'
+                f4m_url += compat_urllib_parse.urlencode(f4m_params).encode('utf-8')
+                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
+                continue
+
+            if src_url.startswith('http'):
+                http_count += 1
+                formats.append({
+                    'url': src_url,
+                    'ext': ext or src_ext or 'flv',
+                    'format_id': 'http-%d' % (bitrate or http_count),
+                    'tbr': bitrate,
+                    'filesize': filesize,
+                    'width': width,
+                    'height': height,
+                })
+                continue
 
         self._sort_formats(formats)
 
         return formats
 
-    def _parse_smil_video(self, video, video_id, base, rtmp_count):
-        src = video.get('src')
-        if not src:
-            return [], rtmp_count
-        bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-        width = int_or_none(video.get('width'))
-        height = int_or_none(video.get('height'))
-        proto = video.get('proto')
-        if not proto:
-            if base:
-                if base.startswith('rtmp'):
-                    proto = 'rtmp'
-                elif base.startswith('http'):
-                    proto = 'http'
-        ext = video.get('ext')
-        if proto == 'm3u8':
-            return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
-        elif proto == 'rtmp':
-            rtmp_count += 1
-            streamer = video.get('streamer') or base
-            return ([{
-                'url': streamer,
-                'play_path': src,
-                'ext': 'flv',
-                'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
-                'tbr': bitrate,
-                'width': width,
-                'height': height,
-            }], rtmp_count)
-        elif proto.startswith('http'):
-            return ([{
-                'url': base + src,
-                'ext': ext or 'flv',
-                'tbr': bitrate,
-                'width': width,
-                'height': height,
-            }], rtmp_count)
+    def _parse_smil_subtitles(self, smil, namespace=None):
+        subtitles = {}
+        for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
+            src = textstream.get('src')
+            if not src:
+                continue
+            ext = textstream.get('ext') or determine_ext(src)
+            if not ext:
+                type_ = textstream.get('type')
+                if type_ == 'text/srt':
+                    ext = 'srt'
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName')
+            subtitles.setdefault(lang, []).append({
+                'url': src,
+                'ext': ext,
+            })
+        return subtitles
 
     def _live_title(self, name):
         """ Generate the title for a live video """

From e5e8d20a3a65832c74b002f247866fcbb92e9246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:13:59 +0600
Subject: [PATCH 1418/1515] [extractor/generic] Improve generic SMIL detection

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8cef61c3c..6900ed96f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1110,11 +1110,13 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed?
+        # Is it an RSS feed or a SMIL file?
         try:
             doc = parse_xml(webpage)
             if doc.tag == 'rss':
                 return self._extract_rss(url, video_id, doc)
+            elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
+                return self._parse_smil(doc, url, video_id)
         except compat_xml_parse_error:
             pass
 

From 308cfe0ab3ec7122602ba2d6a4e3acd2caa7a757 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:14:41 +0600
Subject: [PATCH 1419/1515] [test_downloader] Respect --force-generic-extractor

---
 test/test_download.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 1110357a7..284418834 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -136,7 +136,9 @@ def generator(test_case):
                     # We're not using .download here sine that is just a shim
                     # for outside error handling, and returns the exit code
                     # instead of the result dict.
-                    res_dict = ydl.extract_info(test_case['url'])
+                    res_dict = ydl.extract_info(
+                        test_case['url'],
+                        force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):

From 645f814544f9d40386e504a1eb8cf3558f2c109e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:15:33 +0600
Subject: [PATCH 1420/1515] [test/helper] Allow dicts for mincount

---
 test/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e1129e58f..c8b34654d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -133,8 +133,8 @@ def expect_info_dict(self, got_dict, expected_dict):
             elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
                 got = got_dict.get(info_field)
                 self.assertTrue(
-                    isinstance(got, list),
-                    'Expected field %s to be a list, but it is of type %s' % (
+                    isinstance(got, (list, dict)),
+                    'Expected field %s to be a list or a dict, but it is of type %s' % (
                         info_field, type(got).__name__))
                 expected_num = int(expected.partition(':')[2])
                 assertGreaterEqual(

From 8765222d2211cd6f2a40611249181af0bbb2d531 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:16:21 +0600
Subject: [PATCH 1421/1515] [extractor/generic] Add generic SMIL tests

---
 youtube_dl/extractor/generic.py | 68 +++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6900ed96f..27584c44c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -130,6 +130,74 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
+        # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
+        {
+            'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
+            'info_dict': {
+                'id': 'smil',
+                'ext': 'mp4',
+                'title': 'Automatics, robotics and biocybernetics',
+                'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+                'formats': 'mincount:16',
+                'subtitles': 'mincount:1',
+            },
+            'params': {
+                'force_generic_extractor': True,
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
+        {
+            'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
+            'info_dict': {
+                'id': 'hds',
+                'ext': 'flv',
+                'title': 'hds',
+                'formats': 'mincount:1',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from https://www.restudy.dk/video/play/id/1637
+        {
+            'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
+            'info_dict': {
+                'id': 'video_1637',
+                'ext': 'flv',
+                'title': 'video_1637',
+                'formats': 'mincount:3',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
+        {
+            'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
+            'info_dict': {
+                'id': 'smil-service',
+                'ext': 'flv',
+                'title': 'smil-service',
+                'formats': 'mincount:1',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
+        {
+            'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
+            'info_dict': {
+                'id': '4719370',
+                'ext': 'mp4',
+                'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
+                'formats': 'mincount:3',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 41c3a5a7beebbf5f60c5edb5093d564f0829c5c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:20:49 +0600
Subject: [PATCH 1422/1515] [extractor/common] Fix python 3

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f9578b838..c123d9fca 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1101,7 +1101,7 @@ class InfoExtractor(object):
                         'plugin': 'flowplayer-3.2.0.1',
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
-                f4m_url += compat_urllib_parse.urlencode(f4m_params).encode('utf-8')
+                f4m_url += compat_urllib_parse.urlencode(f4m_params)
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
                 continue
 

From 17712eeb1933f53696c1fc53606174e988a96472 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:31:17 +0600
Subject: [PATCH 1423/1515] [extractor/common] Extract namespace parse routine

---
 youtube_dl/extractor/common.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c123d9fca..717dcec7b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -999,8 +999,7 @@ class InfoExtractor(object):
             assert not fatal
             return []
 
-        namespace = self._search_regex(
-            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+        namespace = self._parse_smil_namespace(smil)
 
         return self._parse_smil_formats(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
@@ -1017,8 +1016,7 @@ class InfoExtractor(object):
             'Unable to download SMIL file', fatal=fatal)
 
     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
-        namespace = self._search_regex(
-            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+        namespace = self._parse_smil_namespace(smil)
 
         formats = self._parse_smil_formats(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
@@ -1045,6 +1043,10 @@ class InfoExtractor(object):
             'subtitles': subtitles,
         }
 
+    def _parse_smil_namespace(self, smil):
+        return self._search_regex(
+            r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
+
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
         base = smil_url
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):

From fa7a1cc5ef52a8dd9a355ab37a74be55ac2ddc1f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 22 Jul 2015 12:34:42 +0100
Subject: [PATCH 1424/1515] [screenwavemedia] fix info extraction (fixes #6270)

Closes #6330.
---
 youtube_dl/extractor/screenwavemedia.py | 84 +++++++++++--------------
 1 file changed, 36 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index d1ab66b32..09c085dcf 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -1,12 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    js_to_json,
 )
 
 
@@ -22,59 +21,48 @@ class ScreenwaveMediaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         playerdata = self._download_webpage(
-            'http://player.screenwavemedia.com/play/player.php?id=%s' % video_id,
+            'http://player.screenwavemedia.com/player.php?id=%s' % video_id,
             video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
             r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
-        vidurl = self._search_regex(
-            r'\'vidurl\'\s*:\s*"([^"]+)"', playerdata, 'vidurl').replace('\\/', '/')
 
-        videolist_url = None
+        playerconfig = self._download_webpage(
+            'http://player.screenwavemedia.com/player.js',
+            video_id, 'Downloading playerconfig webpage')
 
-        mobj = re.search(r"'videoserver'\s*:\s*'(?P<videoserver>[^']+)'", playerdata)
-        if mobj:
-            videoserver = mobj.group('videoserver')
-            mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
-            vidid = mobj.group('vidid') if mobj else video_id
-            videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
-        else:
-            mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
-            if mobj:
-                videolist_url = mobj.group('smil')
+        videoserver = self._search_regex(r"'videoserver'\s*:\s*'([^']+)", playerconfig, 'videoserver')
 
-        if videolist_url:
-            videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
-            formats = []
-            baseurl = vidurl[:vidurl.rfind('/') + 1]
-            for video in videolist.findall('.//video'):
-                src = video.get('src')
-                if not src:
-                    continue
-                file_ = src.partition(':')[-1]
-                width = int_or_none(video.get('width'))
-                height = int_or_none(video.get('height'))
-                bitrate = int_or_none(video.get('system-bitrate'), scale=1000)
-                format = {
-                    'url': baseurl + file_,
-                    'format_id': src.rpartition('.')[0].rpartition('_')[-1],
-                }
-                if width or height:
-                    format.update({
-                        'tbr': bitrate,
-                        'width': width,
-                        'height': height,
-                    })
-                else:
-                    format.update({
-                        'abr': bitrate,
-                        'vcodec': 'none',
-                    })
-                formats.append(format)
-        else:
-            formats = [{
-                'url': vidurl,
-            }]
+        sources = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                    'sources',
+                ).replace(
+                    "' + thisObj.options.videoserver + '",
+                    videoserver
+                ).replace(
+                    "' + playerVidId + '",
+                    video_id
+                )
+            ),
+            video_id
+        )
+
+        formats = []
+        for source in sources:
+            if source['type'] == 'hls':
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+            else:
+                format_label = source.get('label')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_label, 'height', default=None))
+                formats.append({
+                    'url': source['file'],
+                    'format': format_label,
+                    'ext': source.get('type'),
+                    'height': height,
+                })
         self._sort_formats(formats)
 
         return {

From 9cc93c64aa321260475a2bdf7d8626cdd16bf8ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 1 Aug 2015 22:15:43 +0200
Subject: [PATCH 1425/1515] [screenwavemedia] Use the IP for the videoserver
 (fixes #6397)

For http://cinemassacre.com/2015/07/28/avgn-seaman-for-dreamcast/ the other server returns a 403 error.
---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 09c085dcf..3bc84989e 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -31,7 +31,7 @@ class ScreenwaveMediaIE(InfoExtractor):
             'http://player.screenwavemedia.com/player.js',
             video_id, 'Downloading playerconfig webpage')
 
-        videoserver = self._search_regex(r"'videoserver'\s*:\s*'([^']+)", playerconfig, 'videoserver')
+        videoserver = self._search_regex(r"\[ipaddress\]\s*=>\s*([\d\.]+)", playerdata, 'videoserver')
 
         sources = self._parse_json(
             js_to_json(

From cdc682d5a467b7188eb13b5eeb76eb5dd544d1f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 04:21:16 +0600
Subject: [PATCH 1426/1515] [nowtv] Fix extraction (Closes #6357)

---
 youtube_dl/extractor/nowtv.py | 63 +++++++++++++++--------------------
 1 file changed, 27 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 0b5ff4760..de6bc6d96 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     ExtractorError,
+    determine_ext,
     int_or_none,
     parse_iso8601,
     parse_duration,
@@ -15,7 +16,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?P<station>rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
 
     _TESTS = [{
         # rtl
@@ -23,7 +24,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203519',
             'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Die neuen Bauern und eine Hochzeit',
             'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -32,7 +33,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2786,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -41,7 +42,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203481',
             'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Berlin - Tag & Nacht (Folge 934)',
             'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -50,7 +51,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2641,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -59,7 +60,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '165780',
             'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hals- und Beinbruch',
             'description': 'md5:b50d248efffe244e6f56737f0911ca57',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -68,7 +69,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2742,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -77,7 +78,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '99205',
             'display_id': 'medicopter-117/angst',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Angst!',
             'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -86,7 +87,7 @@ class NowTVIE(InfoExtractor):
             'duration': 3025,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -95,7 +96,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203521',
             'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
             'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -104,7 +105,7 @@ class NowTVIE(InfoExtractor):
             'duration': 1083,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -113,7 +114,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '128953',
             'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': "Büro-Fall / Chihuahua 'Joel'",
             'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -122,15 +123,13 @@ class NowTVIE(InfoExtractor):
             'duration': 3092,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-        station = mobj.group('station')
+        display_id = self._match_id(url)
 
         info = self._download_json(
             'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
@@ -148,29 +147,19 @@ class NowTVIE(InfoExtractor):
                 raise ExtractorError(
                     'Video %s is not available for free' % video_id, expected=True)
 
-        f = info.get('format', {})
-        station = f.get('station') or station
-
-        STATIONS = {
-            'rtl': 'rtlnow',
-            'rtl2': 'rtl2now',
-            'vox': 'voxnow',
-            'nitro': 'rtlnitronow',
-            'ntv': 'n-tvnow',
-            'superrtl': 'superrtlnow'
-        }
-
         formats = []
         for item in files['items']:
-            item_path = remove_start(item['path'], '/')
-            tbr = int_or_none(item['bitrate'])
-            m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)
-            m3u8_url = m3u8_url.replace('now/', 'now/videos/')
+            if determine_ext(item['path']) != 'f4v':
+                continue
+            app, play_path = remove_start(item['path'], '/').split('/', 1)
             formats.append({
-                'url': m3u8_url,
-                'format_id': '%s-%sk' % (item['id'], tbr),
-                'ext': 'mp4',
-                'tbr': tbr,
+                'url': 'rtmpe://fms.rtl.de',
+                'app': app,
+                'play_path': 'mp4:%s' % play_path,
+                'ext': 'flv',
+                'page_url': url,
+                'player_url': 'http://rtl-now.rtl.de/includes/nc_player.swf',
+                'tbr': int_or_none(item.get('bitrate')),
             })
         self._sort_formats(formats)
 
@@ -178,6 +167,8 @@ class NowTVIE(InfoExtractor):
         description = info.get('articleLong') or info.get('articleShort')
         timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
         duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
         thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
 
         return {

From e422d7f4f78994de8483d2207ab4e00174a2408c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 04:26:59 +0600
Subject: [PATCH 1427/1515] [nowtv] Expand _VALID_URL

---
 youtube_dl/extractor/nowtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index de6bc6d96..11ce37168 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -126,6 +126,9 @@ class NowTVIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From d41d04c0f513ad3b83ab6aee60cf2201710b6063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 06:35:35 +0600
Subject: [PATCH 1428/1515] [videolectures] Fix _VALID_URL

---
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index d6a7eb203..24584dc80 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)(?:/?[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
     _TEST = {

From 5c45bbe57bd791debfd64052ab030298a7c6b718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Aug 2015 15:19:30 +0200
Subject: [PATCH 1429/1515] [nowtv] Remove unused import

---
 youtube_dl/extractor/nowtv.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 11ce37168..ad938fb62 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (

From 25a4c5a9ed59eca0241922363e83e61172527658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Aug 2015 15:19:57 +0200
Subject: [PATCH 1430/1515] [dailymotion:playlist] Use an iterator for the
 entries

So that using '--playlist-end' only downloads the required pages (reported in #2175).
---
 youtube_dl/extractor/dailymotion.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 85d945509..2d90b2224 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -15,7 +15,6 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
-    orderedSet,
     parse_iso8601,
     str_to_int,
     unescapeHTML,
@@ -278,7 +277,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     }]
 
     def _extract_entries(self, id):
-        video_ids = []
+        video_ids = set()
         processed_urls = set()
         for pagenum in itertools.count(1):
             page_url = self._PAGE_TEMPLATE % (id, pagenum)
@@ -291,12 +290,13 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
             processed_urls.add(urlh.geturl())
 
-            video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
+            for video_id in re.findall(r'data-xid="(.+?)"', webpage):
+                if video_id not in video_ids:
+                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
+                    video_ids.add(video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
                 break
-        return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
-                for video_id in orderedSet(video_ids)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From d7d2a9a3dbf1cef78c5085a4aab5d2f336c64cff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 03:28:04 +0100
Subject: [PATCH 1431/1515] [utils] restart download if server does not support
 byte ranges

---
 youtube_dl/downloader/http.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b7f144af9..b2e82cfde 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -57,6 +57,20 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
+
+                if resume_len > 0:
+                    content_range = data.headers.get('Content-Range')
+                    if content_range:
+                        content_range_m = re.search(r'bytes (\d+)-', content_range)
+                        if content_range_m:
+                            # Content-Range is correct - go on
+                            if resume_len == int(content_range_m.group(1)):
+                                break
+
+                    # Content-Range is invalid - wipe the file and do entire redownload
+                    resume_len = 0
+                    open_mode = 'wb'
+
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:

From 8d5b8b477e4b1051482b21ea451f0de1ce23bce7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 03:58:02 +0100
Subject: [PATCH 1432/1515] [utils] import re

---
 youtube_dl/downloader/http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b2e82cfde..f796ee113 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -4,6 +4,7 @@ import errno
 import os
 import socket
 import time
+import re
 
 from .common import FileDownloader
 from ..compat import (

From c3124c3085e6a9a83ee31ace3a7d528a324c42da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:25:08 +0600
Subject: [PATCH 1433/1515] [downloader/http] Simplify

---
 youtube_dl/downloader/http.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f796ee113..0862e90bb 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,20 +58,16 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
-
                 if resume_len > 0:
                     content_range = data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
-                        if content_range_m:
-                            # Content-Range is correct - go on
-                            if resume_len == int(content_range_m.group(1)):
-                                break
-
+                        # Content-Range is correct - go on
+                    if content_range_m and resume_len == int(content_range_m.group(1)):
+                        break
                     # Content-Range is invalid - wipe the file and do entire redownload
                     resume_len = 0
                     open_mode = 'wb'
-
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:

From 10eaa8ef1d2a9699052af9262aa472456548e99b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:25:40 +0600
Subject: [PATCH 1434/1515] [downloader/http] Report unable to resume

---
 youtube_dl/downloader/http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 0862e90bb..2f8490f02 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -66,6 +66,7 @@ class HttpFD(FileDownloader):
                     if content_range_m and resume_len == int(content_range_m.group(1)):
                         break
                     # Content-Range is invalid - wipe the file and do entire redownload
+                    self.report_unable_to_resume()
                     resume_len = 0
                     open_mode = 'wb'
                 break

From 84bc4dcb0f678f0a8c9f993e101b9769e3959f76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:27:47 +0600
Subject: [PATCH 1435/1515] [downloader/http] Clarify rationale for
 Content-Range check (#6426)

---
 youtube_dl/downloader/http.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2f8490f02..a29f5cf31 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,14 +58,21 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
+                # When trying to resume, Content-Range HTTP header of response has to be checked
+                # to match the value of requested Range HTTP header. This is due to a webservers
+                # that don't support resuming and serve a whole file with no Content-Range
+                # set in response despite of requested Range (see
+                # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
                 if resume_len > 0:
                     content_range = data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
-                        # Content-Range is correct - go on
-                    if content_range_m and resume_len == int(content_range_m.group(1)):
-                        break
-                    # Content-Range is invalid - wipe the file and do entire redownload
+                        # Content-Range is present and matches requested Range, resume is possible
+                        if content_range_m and resume_len == int(content_range_m.group(1)):
+                            break
+                    # Content-Range is either not present or invalid. Assuming remote webserver is
+                    # trying to send the whole file, resume is not possible, so wiping the local file
+                    # and performing entire redownload
                     self.report_unable_to_resume()
                     resume_len = 0
                     open_mode = 'wb'

From 754e70cf3e74218ae5d840985fbf07bbe274332a Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sun, 2 Aug 2015 19:21:10 +0100
Subject: [PATCH 1436/1515] [pornhub] Fix video url regular expression.

PornHub seems to have subtly changed their JavaScript. Before, video URL strings were embedded directly in the video's `flashvars_*` object, but they are now assigned to variables of the form `player_quality_*`, which are then added to this object later under the relevant quality key.
---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 0b7886840..fbaa830d6 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -81,7 +81,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"var player_quality_[0-9]{3}p = '([^']+)'", webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From 524229a2975c20887a9a71cae77132e775003537 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:41:17 +0600
Subject: [PATCH 1437/1515] [pornhub] Improve

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fbaa830d6..fec493046 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -81,7 +81,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"var player_quality_[0-9]{3}p = '([^']+)'", webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"player_quality_[0-9]{3}p\s*=\s*'([^']+)'", webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From 51a575159a5a83e4477b03544f419dcf2e9ff0fa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 22:52:12 +0100
Subject: [PATCH 1438/1515] [facebook] extract uploader

---
 youtube_dl/extractor/facebook.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index e17bb9aea..734de4da2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -17,6 +17,8 @@ from ..utils import (
     int_or_none,
     limit_length,
     urlencode_postdata,
+    get_element_by_id,
+    clean_html,
 )
 
 
@@ -161,6 +163,7 @@ class FacebookIE(InfoExtractor):
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id
+        uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
 
         return {
             'id': video_id,
@@ -168,4 +171,5 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'duration': int_or_none(video_data.get('video_duration')),
             'thumbnail': video_data.get('thumbnail_src'),
+            'uploader': uploader,
         }

From 67b8a28a2f69764259cf2e90c0a3785c05c55551 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 3 Aug 2015 00:09:21 +0100
Subject: [PATCH 1439/1515] [facebook] add uploader value to the tests

---
 youtube_dl/extractor/facebook.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 734de4da2..178a7ca4c 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -44,6 +44,7 @@ class FacebookIE(InfoExtractor):
             'id': '637842556329505',
             'ext': 'mp4',
             'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
+            'uploader': 'Tennis on Facebook',
         }
     }, {
         'note': 'Video without discernible title',
@@ -52,6 +53,7 @@ class FacebookIE(InfoExtractor):
             'id': '274175099429670',
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
+            'uploader': 'Asif Nawab Butt',
         },
         'expected_warnings': [
             'title'

From 8de922724b8f3ad31ff7249799de371ff8a5c3ad Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 05:36:17 +0600
Subject: [PATCH 1440/1515] [README.md] Clarify using cookies

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index ac54d7b67..2db3139ee 100644
--- a/README.md
+++ b/README.md
@@ -439,6 +439,12 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
     youtube-dl -- -wNyEUrxzFU
     youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
 
+### How do I pass cookies to youtube-dl?
+
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that cookies file must be in Mozilla/Netscape format and the first line of cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in cookies file and convert newlines if necessary to correspond your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+
+Passing cookies to youtube-dl is a good way to workaround login when particular extractor does not implement it explicitly.
+
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
 As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.

From 47a8b7c14a085ce558db3b5a85ded850cd5df642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 3 Aug 2015 12:00:08 +0200
Subject: [PATCH 1441/1515] [mdr] Change XPath to make it work in python 2.6
 (fixes #6443)

The 'progressiveDownloadUrl' element is a direct child, so they should be equivalent.
---
 youtube_dl/extractor/mdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 5fdd19027..fc7499958 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -29,7 +29,7 @@ class MDRIE(InfoExtractor):
         doc = self._download_xml(domain + xmlurl, video_id)
         formats = []
         for a in doc.findall('./assets/asset'):
-            url_el = a.find('.//progressiveDownloadUrl')
+            url_el = a.find('./progressiveDownloadUrl')
             if url_el is None:
                 continue
             abr = int(a.find('bitrateAudio').text) // 1000

From 8f5639afcbb967f276fb8b35a24559cdcc3b6d32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 19:37:48 +0600
Subject: [PATCH 1442/1515] [pornhub] Improve video quality regex

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fec493046..7b0cdc41a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -94,7 +94,7 @@ class PornHubIE(InfoExtractor):
             format = path.split('/')[5].split('_')[:2]
             format = "-".join(format)
 
-            m = re.match(r'^(?P<height>[0-9]+)P-(?P<tbr>[0-9]+)K$', format)
+            m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
             if m is None:
                 height = None
                 tbr = None

From e704f87f869b98bbed56d7dd0fe27710306c8272 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@nand.wakku.to>
Date: Mon, 3 Aug 2015 01:54:21 +0200
Subject: [PATCH 1443/1515] [twitch] Parse start_time from 't' (closes #6441)

Eg. for VOD links like http://www.twitch.tv/gamesdonequick/v/9136645?t=14h29m15s
---
 youtube_dl/extractor/twitch.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 73ce335b7..a2b6a35aa 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -7,12 +7,15 @@ import random
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_str,
     compat_urllib_parse,
+    compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    parse_duration,
     parse_iso8601,
 )
 
@@ -185,7 +188,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_SHORTCUT = 'v'
 
     _TEST = {
-        'url': 'http://www.twitch.tv/riotgames/v/6528877',
+        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
         'info_dict': {
             'id': 'v6528877',
             'ext': 'mp4',
@@ -197,6 +200,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             'uploader': 'Riot Games',
             'uploader_id': 'riotgames',
             'view_count': int,
+            'start_time': 310,
         },
         'params': {
             # m3u8 download
@@ -216,6 +220,12 @@ class TwitchVodIE(TwitchItemBaseIE):
             item_id, 'mp4')
         self._prefer_source(formats)
         info['formats'] = formats
+
+        parsed_url = compat_urllib_parse_urlparse(url)
+        query = compat_parse_qs(parsed_url.query)
+        if 't' in query:
+            info['start_time'] = parse_duration(query['t'][0])
+
         return info
 
 

From d96d604e5311628ece0234733dbbfe73a58c8d18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 3 Aug 2015 23:04:11 +0200
Subject: [PATCH 1444/1515] YoutubeDL: format spec: don't accept a bare '/'
 (#6124)

---
 test/test_YoutubeDL.py  | 1 +
 youtube_dl/YoutubeDL.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 20f45f439..9a3c28f8c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -311,6 +311,7 @@ class TestFormatSelection(unittest.TestCase):
         assert_syntax_error('bestvideo,,best')
         assert_syntax_error('+bestaudio')
         assert_syntax_error('bestvideo+')
+        assert_syntax_error('/')
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index efa3254ce..c608ff91a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -960,6 +960,8 @@ class YoutubeDL(object):
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
+                        if not current_selector:
+                            raise syntax_error('"/" must follow a format selector', start)
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])

From a346b1ff57a94382e80fd4edd5a6d4b91a7cb45e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Aug 2015 20:44:22 +0600
Subject: [PATCH 1445/1515] [bbc] Add support for vxp-playlist-data embeds
 (Closes #6453)

---
 youtube_dl/extractor/bbc.py | 45 ++++++++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 9a1b6e3dc..abc5a44a1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -526,6 +526,18 @@ class BBCIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video from video playlist embedded with vxp-playlist-data JSON
+        'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
+        'info_dict': {
+            'id': 'p02w6qjc',
+            'ext': 'mp4',
+            'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
+            'duration': 56,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         # single video story with digitalData
         'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
@@ -695,13 +707,36 @@ class BBCIE(BBCCoUkIE):
 
         if not medias:
             # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
-            media_asset_page = self._parse_json(
+            media_asset = self._search_regex(
+                r'mediaAssetPage\.init\(\s*({.+?}), "/',
+                webpage, 'media asset', default=None)
+            if media_asset:
+                media_asset_page = self._parse_json(media_asset, playlist_id, fatal=False)
+                medias = []
+                for video in media_asset_page.get('videos', {}).values():
+                    medias.extend(video.values())
+
+        if not medias:
+            # Multiple video playlist with single `now playing` entry (e.g.
+            # http://www.bbc.com/news/video_and_audio/must_see/33767813)
+            vxp_playlist = self._parse_json(
                 self._search_regex(
-                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                    r'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>',
+                    webpage, 'playlist data'),
                 playlist_id)
-            medias = []
-            for video in media_asset_page.get('videos', {}).values():
-                medias.extend(video.values())
+            playlist_medias = []
+            for item in vxp_playlist:
+                media = item.get('media')
+                if not media:
+                    continue
+                playlist_medias.append(media)
+                # Download single video if found media with asset id matching the video id from URL
+                if item.get('advert', {}).get('assetId') == playlist_id:
+                    medias = [media]
+                    break
+            # Fallback to the whole playlist
+            if not medias:
+                medias = playlist_medias
 
         entries = []
         for num, media_meta in enumerate(medias, start=1):

From 232541df441741d3d55605f03e28ec3c34249a5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 4 Aug 2015 22:29:23 +0200
Subject: [PATCH 1446/1515] [YoutubeDL] format spec: correctly handle dashes
 and other unused operators

'mp4-baseline-16x9' must be handled as a single string, but the '-' was treated as an operator.
---
 test/test_YoutubeDL.py  |  6 ++++++
 youtube_dl/YoutubeDL.py | 33 ++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 9a3c28f8c..0388c0bf3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -105,6 +105,7 @@ class TestFormatSelection(unittest.TestCase):
     def test_format_selection(self):
         formats = [
             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
             {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
             {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
             {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
@@ -136,6 +137,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], '35')
 
+        ydl = YDL({'format': 'example-with-dashes'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'example-with-dashes')
+
     def test_format_selection_audio(self):
         formats = [
             {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c608ff91a..1446b3254 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -933,6 +933,37 @@ class YoutubeDL(object):
                 else:
                     filter_parts.append(string)
 
+        def _remove_unused_ops(tokens):
+            # Remove operators that we don't use and join them with the sourrounding strings
+            # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
+            ALLOWED_OPS = ('/', '+', ',', '(', ')')
+            last_string, last_start, last_end, last_line = None, None, None, None
+            for type, string, start, end, line in tokens:
+                if type == tokenize.OP and string == '[':
+                    if last_string:
+                        yield tokenize.NAME, last_string, last_start, last_end, last_line
+                        last_string = None
+                    yield type, string, start, end, line
+                    # everything inside brackets will be handled by _parse_filter
+                    for type, string, start, end, line in tokens:
+                        yield type, string, start, end, line
+                        if type == tokenize.OP and string == ']':
+                            break
+                elif type == tokenize.OP and string in ALLOWED_OPS:
+                    if last_string:
+                        yield tokenize.NAME, last_string, last_start, last_end, last_line
+                        last_string = None
+                    yield type, string, start, end, line
+                elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
+                    if not last_string:
+                        last_string = string
+                        last_start = start
+                        last_end = end
+                    else:
+                        last_string += string
+            if last_string:
+                yield tokenize.NAME, last_string, last_start, last_end, last_line
+
         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
@@ -1111,7 +1142,7 @@ class YoutubeDL(object):
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
         try:
-            tokens = list(compat_tokenize_tokenize(stream.readline))
+            tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
         except tokenize.TokenError:
             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
 

From 8a5601e42f6974e6694f01089b4c7e014b6a1b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 19:52:04 +0600
Subject: [PATCH 1447/1515] [lynda] Fix login (Closes #6462)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index a00f6e5e5..39214de2f 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
+    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn\s*:\s*true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 

From 5b7dab2dd640c93ec0f63ca8b901e701679a4c7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:06:48 +0600
Subject: [PATCH 1448/1515] [lynda] Make login more robust

---
 youtube_dl/extractor/lynda.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 39214de2f..deead220a 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -17,7 +17,6 @@ from ..utils import (
 
 class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn\s*:\s*true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 
@@ -41,7 +40,7 @@ class LyndaBaseIE(InfoExtractor):
             request, None, 'Logging in as %s' % username)
 
         # Not (yet) logged in
-        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
+        m = re.search(r'loginResultJson\s*=\s*\'(?P<json>[^\']+)\';', login_page)
         if m is not None:
             response = m.group('json')
             response_json = json.loads(response)
@@ -70,7 +69,7 @@ class LyndaBaseIE(InfoExtractor):
                     request, None,
                     'Confirming log in and log out from another device')
 
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+        if all(not re.search(p, login_page) for p in ('isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
             raise ExtractorError('Unable to log in')
 
 

From 354b4b8604ec13ccf4bd89b9d1b77cb7246fe379 Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Wed, 5 Aug 2015 19:37:59 +0530
Subject: [PATCH 1449/1515] fixing xhamster file extraction

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b4ad513a0..9d025530f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -47,7 +47,7 @@ class XHamsterIE(InfoExtractor):
 
     def _real_extract(self, url):
         def extract_video_url(webpage):
-            mp4 = re.search(r'<video\s+.*?file="([^"]+)".*?>', webpage)
+            mp4 = re.search(r'file:\s+\'([^\']+)\'', webpage)
             if mp4 is None:
                 raise ExtractorError('Unable to extract media URL')
             else:

From be7a8379b47c35afe66abcc02aee597e5143b1d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:32:44 +0600
Subject: [PATCH 1450/1515] [xhamster] Make more robust

---
 youtube_dl/extractor/xhamster.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 9d025530f..481d79b89 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -46,12 +46,12 @@ class XHamsterIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        def extract_video_url(webpage):
-            mp4 = re.search(r'file:\s+\'([^\']+)\'', webpage)
-            if mp4 is None:
-                raise ExtractorError('Unable to extract media URL')
-            else:
-                return mp4.group(1)
+        def extract_video_url(webpage, name):
+            return self._search_regex(
+                [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
+                 r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
+                 r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
+                webpage, name, group='mp4')
 
         def is_hd(webpage):
             return '<div class=\'icon iconHD\'' in webpage
@@ -97,7 +97,9 @@ class XHamsterIE(InfoExtractor):
 
         hd = is_hd(webpage)
 
-        video_url = extract_video_url(webpage)
+        format_id = 'hd' if hd else 'sd'
+
+        video_url = extract_video_url(webpage, format_id)
         formats = [{
             'url': video_url,
             'format_id': 'hd' if hd else 'sd',
@@ -108,7 +110,7 @@ class XHamsterIE(InfoExtractor):
             mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
             webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
             if is_hd(webpage):
-                video_url = extract_video_url(webpage)
+                video_url = extract_video_url(webpage, 'hd')
                 formats.append({
                     'url': video_url,
                     'format_id': 'hd',

From 251a44b776264c17d7799e017b856143c6cacd9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:36:37 +0600
Subject: [PATCH 1451/1515] [xhamster] Fix thumbnail extraction

---
 youtube_dl/extractor/xhamster.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 481d79b89..b57e7c813 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -78,7 +78,10 @@ class XHamsterIE(InfoExtractor):
         uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
                                               webpage, 'uploader id', default='anonymous')
 
-        thumbnail = self._html_search_regex(r'<video\s+.*?poster="([^"]+)".*?>', webpage, 'thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
+             r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
+             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
                                                           webpage, 'duration', fatal=False))

From 3e4852247744b131600ba43275ab321eb1b32bb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:41:40 +0600
Subject: [PATCH 1452/1515] [xhamster] Fix uploader extraction

---
 youtube_dl/extractor/xhamster.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b57e7c813..06fedf840 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -22,7 +22,7 @@ class XHamsterIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'FemaleAgent Shy beauty takes the bait',
                 'upload_date': '20121014',
-                'uploader_id': 'Ruseful2011',
+                'uploader': 'Ruseful2011',
                 'duration': 893,
                 'age_limit': 18,
             }
@@ -34,7 +34,7 @@ class XHamsterIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Britney Spears  Sexy Booty',
                 'upload_date': '20130914',
-                'uploader_id': 'jojo747400',
+                'uploader': 'jojo747400',
                 'duration': 200,
                 'age_limit': 18,
             }
@@ -75,8 +75,9 @@ class XHamsterIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
-        uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
-                                              webpage, 'uploader id', default='anonymous')
+        uploader = self._html_search_regex(
+            r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",
+            webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
             [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
@@ -127,7 +128,7 @@ class XHamsterIE(InfoExtractor):
             'title': title,
             'description': description,
             'upload_date': upload_date,
-            'uploader_id': uploader_id,
+            'uploader': uploader,
             'thumbnail': thumbnail,
             'duration': duration,
             'view_count': view_count,

From 54a9328b205e8a2c916d59fd81bdb1ede25cf87a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 21:19:52 +0600
Subject: [PATCH 1453/1515] [generic] Expand jwplayer support

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8cef61c3c..6df89f814 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1655,7 +1655,7 @@ class GenericIE(InfoExtractor):
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
             found = filter_video(re.findall(
-                r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
+                r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
         if not found:
             # Flow player
             found = filter_video(re.findall(r'''(?xs)

From c71a3195afa8c2a9ed5fe0ffa56ff6c969147d91 Mon Sep 17 00:00:00 2001
From: Delon <liuxi326@qq.com>
Date: Wed, 5 Aug 2015 18:22:25 +0800
Subject: [PATCH 1454/1515] [tudou] Fix extracion

---
 youtube_dl/extractor/tudou.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c89de5ba4..9b934cb57 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -29,6 +29,8 @@ class TudouIE(InfoExtractor):
         }
     }]
 
+    _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
+	
     def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
@@ -76,6 +78,9 @@ class TudouIE(InfoExtractor):
                 'ext': ext,
                 'title': title,
                 'thumbnail': thumbnail_url,
+                'http_headers': {
+                    'Referer': self._PLAYER_URL,
+                },
             }
             result.append(part_info)
 

From 238755752f4f9169a1edda91067c8627afe19cce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:07:52 +0600
Subject: [PATCH 1455/1515] [tudou] Extract player URL from the webpage

---
 youtube_dl/extractor/tudou.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 9b934cb57..84fe71aef 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -30,7 +30,7 @@ class TudouIE(InfoExtractor):
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
-	
+
     def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
@@ -56,6 +56,10 @@ class TudouIE(InfoExtractor):
         thumbnail_url = self._search_regex(
             r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False)
 
+        player_url = self._search_regex(
+            r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
+            webpage, 'player URL', default=self._PLAYER_URL)
+
         segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
         segments = json.loads(segs_json)
         # It looks like the keys are the arguments that have to be passed as
@@ -79,7 +83,7 @@ class TudouIE(InfoExtractor):
                 'title': title,
                 'thumbnail': thumbnail_url,
                 'http_headers': {
-                    'Referer': self._PLAYER_URL,
+                    'Referer': player_url,
                 },
             }
             result.append(part_info)

From f535ec8278c8f465b47919d3f451571ae8ccfc7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:08:26 +0600
Subject: [PATCH 1456/1515] [xhamster] Remove unused import

---
 youtube_dl/extractor/xhamster.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 06fedf840..f76ee8fd4 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,7 +4,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     unified_strdate,
     str_to_int,
     int_or_none,

From c73cdd800f0dc7b465ac0b36d338875bb80c23aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:08:55 +0600
Subject: [PATCH 1457/1515] [xhamster] flake8

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f76ee8fd4..97315750f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -81,7 +81,7 @@ class XHamsterIE(InfoExtractor):
         thumbnail = self._search_regex(
             [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
              r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
-             webpage, 'thumbnail', fatal=False, group='thumbnail')
+            webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
                                                           webpage, 'duration', fatal=False))

From 51f267d9d4d26c3cd67f318a2040513946f2b4d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 22:01:01 +0600
Subject: [PATCH 1458/1515] [YoutubeDL:utils] Move percent encode non-ASCII
 URLs workaround to http_request and simplify (Closes #6457)

---
 youtube_dl/YoutubeDL.py | 21 ---------------------
 youtube_dl/utils.py     | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1446b3254..079d42ce8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1860,27 +1860,6 @@ class YoutubeDL(object):
 
     def urlopen(self, req):
         """ Start an HTTP download """
-
-        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
-        # always respected by websites, some tend to give out URLs with non percent-encoded
-        # non-ASCII characters (see telemb.py, ard.py [#3412])
-        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
-        # To work around aforementioned issue we will replace request's original URL with
-        # percent-encoded one
-        req_is_string = isinstance(req, compat_basestring)
-        url = req if req_is_string else req.get_full_url()
-        url_escaped = escape_url(url)
-
-        # Substitute URL if any change after escaping
-        if url != url_escaped:
-            if req_is_string:
-                req = url_escaped
-            else:
-                req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
-                req = req_type(
-                    url_escaped, data=req.data, headers=req.headers,
-                    origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
-
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 78dc2b449..c7db75f80 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -651,6 +651,26 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         return ret
 
     def http_request(self, req):
+        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
+        # always respected by websites, some tend to give out URLs with non percent-encoded
+        # non-ASCII characters (see telemb.py, ard.py [#3412])
+        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
+        # To work around aforementioned issue we will replace request's original URL with
+        # percent-encoded one
+        # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
+        # the code of this workaround has been moved here from YoutubeDL.urlopen()
+        url = req.get_full_url()
+        url_escaped = escape_url(url)
+
+        # Substitute URL if any change after escaping
+        if url != url_escaped:
+            req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+            new_req = req_type(
+                url_escaped, data=req.data, headers=req.headers,
+                origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+            new_req.timeout = req.timeout
+            req = new_req
+
         for h, v in std_headers.items():
             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
             # The dict keys are capitalized because of this bug by urllib

From bd690a9f9368095f561184778fb2f3ef12c66342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 22:01:31 +0600
Subject: [PATCH 1459/1515] [southpark:de] Add test for non-ASCII in URLs

---
 youtube_dl/extractor/southpark.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 7fb165a87..87b650468 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -45,6 +45,14 @@ class SouthParkDeIE(SouthParkIE):
             'title': 'The Government Won\'t Respect My Privacy',
             'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
         },
+    }, {
+        # non-ASCII characters in initial URL
+        'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
+        'playlist_count': 4,
+    }, {
+        # non-ASCII characters in redirect URL
+        'url': 'http://www.southpark.de/alle-episoden/s18e09',
+        'playlist_count': 4,
     }]
 
 

From 4f34cdb0a87a506d25a352ff265678c86cb9b979 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 23:56:44 +0600
Subject: [PATCH 1460/1515] [southpark:de] Skip test

---
 youtube_dl/extractor/southpark.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 87b650468..ad63a8785 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -53,6 +53,7 @@ class SouthParkDeIE(SouthParkIE):
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
         'playlist_count': 4,
+        'skip': 'Broken python 3',
     }]
 
 

From 671302b5c0ff8cefa5f26e599423ef7799b19631 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 00:08:11 +0600
Subject: [PATCH 1461/1515] [YoutubeDL] Remove unused imports

---
 youtube_dl/YoutubeDL.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 079d42ce8..cad6b026e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,7 +28,6 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
-    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_get_terminal_size,
@@ -40,7 +39,6 @@ from .compat import (
     compat_urllib_request,
 )
 from .utils import (
-    escape_url,
     ContentTooShortError,
     date_from_str,
     DateRange,
@@ -51,7 +49,6 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
-    HEADRequest,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,

From cd6b555e19c601d575679dd29da0080eda7f8890 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 6 Aug 2015 19:17:50 +0100
Subject: [PATCH 1462/1515] [dcn] add origin to api request and fix the test
 and check with flake8

---
 youtube_dl/extractor/dcn.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index f76ebda9e..d44e8cef0 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -1,4 +1,9 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
+
 
 class DcnIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
@@ -9,24 +14,29 @@ class DcnIE(InfoExtractor):
             'id': '17375',
             'ext': 'm3u8',
             'title': 'رحلة العمر : الحلقة 1',
-            'description': '"في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة1"',
+            'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
             'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
             'duration': '2041'
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json_data = self._download_json(
-            'http://admin.mangomolo.com/analytics/index.php/plus/video?id='+video_id,
-            video_id
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=' + video_id,
+            headers={'Origin': 'http://www.dcndigital.ae'}
         )
-        title = json_data['title_ar'];
-        thumbnail = 'http://admin.mangomolo.com/analytics/'+json_data['img'];
-        duration = json_data['duration'];
-        description = json_data['description_ar'];
+        json_data = self._download_json(request, video_id)
+        title = json_data['title_ar']
+        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data['img']
+        duration = json_data['duration']
+        description = json_data['description_ar']
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id='+json_data['id']+'&user_id='+json_data['user_id']+'&countries=Q0M=&w=100%&h=100%&filter=DENY&signature='+json_data['signature'],
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
             video_id
         )
         m3u8_url = self._html_search_regex(

From 5a4d9ddb218e761fe7ab15d197690e0cb132a536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 01:26:40 +0600
Subject: [PATCH 1463/1515] [utils] Percent-encode redirect URL of Location
 header (Closes #6457)

---
 youtube_dl/utils.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c7db75f80..e265c7574 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -715,6 +715,17 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
+        if 300 <= resp.code < 400:
+            location = resp.headers.get('Location')
+            if location:
+                # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
+                if sys.version_info >= (3, 0):
+                    location = location.encode('iso-8859-1').decode('utf-8')
+                location_escaped = escape_url(location)
+                if location != location_escaped:
+                    del resp.headers['Location']
+                    resp.headers['Location'] = location_escaped
         return resp
 
     https_request = http_request

From 9663bd3abb78911bddad75742bd41006677d628e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 01:27:07 +0600
Subject: [PATCH 1464/1515] [southpark:de] Enable non-ASCII redirect URL test

---
 youtube_dl/extractor/southpark.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index ad63a8785..87b650468 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -53,7 +53,6 @@ class SouthParkDeIE(SouthParkIE):
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
         'playlist_count': 4,
-        'skip': 'Broken python 3',
     }]
 
 

From 3eb5fdb58112032a9831eda1d2e3b8a151ea217f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 6 Aug 2015 22:55:43 +0200
Subject: [PATCH 1465/1515] release 2015.08.06

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fa157cadb..b81d5e658 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.28'
+__version__ = '2015.08.06'

From 430b092a5f59fbe407b92ebcb0c42b9f7062a334 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 6 Aug 2015 23:06:21 +0200
Subject: [PATCH 1466/1515] release 2015.08.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b81d5e658..9f209499c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.06'
+__version__ = '2015.08.06.1'

From 6d30cf04db9c9662dbb30c2490e24eb5c6dca4c3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 7 Aug 2015 10:01:18 +0100
Subject: [PATCH 1467/1515] [dcn] fix type and key errors

---
 youtube_dl/extractor/dcn.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index d44e8cef0..22ff35b56 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
+from ..utils import int_or_none
 
 
 class DcnIE(InfoExtractor):
@@ -16,7 +17,7 @@ class DcnIE(InfoExtractor):
             'title': 'رحلة العمر : الحلقة 1',
             'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
             'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
-            'duration': '2041'
+            'duration': 2041
         },
         'params': {
             # m3u8 download
@@ -32,9 +33,9 @@ class DcnIE(InfoExtractor):
         )
         json_data = self._download_json(request, video_id)
         title = json_data['title_ar']
-        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data['img']
-        duration = json_data['duration']
-        description = json_data['description_ar']
+        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data.get('img')
+        duration = int_or_none(json_data.get('duration'))
+        description = json_data.get('description_ar')
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
             video_id

From 8002ac9e0a88d918735c06599dbf8f2005f79666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:04:44 +0600
Subject: [PATCH 1468/1515] [nowtv] Add support for .at TLD

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index ad938fb62..78e8851c0 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl

From acc1adbe7ab93657cd4d303cee1fba4464931a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:50:54 +0600
Subject: [PATCH 1469/1515] [nowtv] Add support for .ch TLD

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 78e8851c0..fc21d8e3f 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl

From 0f422256d6eea5aff062a4c35d7434cd118c7a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:51:09 +0600
Subject: [PATCH 1470/1515] [nowtv] Add .at test

---
 youtube_dl/extractor/nowtv.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index fc21d8e3f..66c627bec 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -127,6 +127,9 @@ class NowTVIE(InfoExtractor):
     }, {
         'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f94639fadf91312bf3365802981f506ecba698dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 00:06:03 +0600
Subject: [PATCH 1471/1515] [dcn] Improve

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/dcn.py      | 78 ++++++++++++++++++++++----------
 2 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eb8ef1fe3..922d9b3d8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -118,7 +118,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
-from .dcn import DcnIE
+from .dcn import DCNIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 22ff35b56..b98a6c032 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,22 +2,30 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import int_or_none
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
-class DcnIE(InfoExtractor):
+class DCNIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
         'info_dict':
         {
             'id': '17375',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'رحلة العمر : الحلقة 1',
-            'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
-            'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
-            'duration': 2041
+            'description': 'md5:0156e935d870acb8ef0a66d24070c6d6',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2041,
+            'timestamp': 1227504126,
+            'upload_date': '20081124',
         },
         'params': {
             # m3u8 download
@@ -27,30 +35,50 @@ class DcnIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         request = compat_urllib_request.Request(
-            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=' + video_id,
-            headers={'Origin': 'http://www.dcndigital.ae'}
-        )
-        json_data = self._download_json(request, video_id)
-        title = json_data['title_ar']
-        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data.get('img')
-        duration = int_or_none(json_data.get('duration'))
-        description = json_data.get('description_ar')
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
+            headers={'Origin': 'http://www.dcndigital.ae'})
+
+        video = self._download_json(request, video_id)
+        title = video.get('title_en') or video['title_ar']
+
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
-            video_id
-        )
-        m3u8_url = self._html_search_regex(
-            r'file:\s*"([^"]+)',
-            webpage,
-            'm3u8_url'
-        )
-        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
+            + compat_urllib_parse.urlencode({
+                'id': video['id'],
+                'user_id': video['user_id'],
+                'signature': video['signature'],
+                'countries': 'Q0M=',
+                'filter': 'DENY',
+            }), video_id)
+
+        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+
+        img = video.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video.get('duration'))
+        description = video.get('description_en') or video.get('description_ar')
+        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
+
         return {
             'id': video_id,
             'title': title,
+            'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'description': description,
+            'timestamp': timestamp,
             'formats': formats,
         }

From 4a7434d0b09e14b773c2d278c8299efa6225b84e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 00:19:40 +0600
Subject: [PATCH 1472/1515] [dcn] Simplify _VALID_URL

---
 youtube_dl/extractor/dcn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index b98a6c032..82261e25c 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class DCNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
         'info_dict':

From fd5d8270dcd6d8baada3390a4a1cae5bdbcb6da4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 01:10:41 +0600
Subject: [PATCH 1473/1515] [clipfish] Fix extraction, minimize requests, get
 rid of drm hds, extract m3u8 and more metadata

---
 youtube_dl/extractor/clipfish.py | 56 ++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 09dfaac60..7af903571 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,18 +1,19 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     int_or_none,
     js_to_json,
-    determine_ext,
+    parse_iso8601,
+    remove_end,
 )
 
 
 class ClipfishIE(InfoExtractor):
-    IE_NAME = 'clipfish'
-
-    _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
         'md5': '79bc922f3e8a9097b3d68a93780fd475',
@@ -20,35 +21,48 @@ class ClipfishIE(InfoExtractor):
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
+            'timestamp': 1370938118,
+            'upload_date': '20130611',
             'duration': 82,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_info = self._parse_json(
-            js_to_json(self._html_search_regex('var videoObject = ({[^}]+?})', webpage, 'videoObject')),
-            video_id
-        )
-        info_url = self._parse_json(
-            js_to_json(self._html_search_regex('var globalFlashvars = ({[^}]+?})', webpage, 'globalFlashvars')),
-            video_id
-        )['data']
 
-        doc = self._download_xml(
-            info_url, video_id, note='Downloading info page')
-        title = doc.find('title').text
-        video_url = doc.find('filename').text
-        thumbnail = doc.find('imageurl').text
-        duration = int_or_none(video_info['length'])
-        formats = [{'url': video_info['videourl']},{'url': video_url}]
+        webpage = self._download_webpage(url, video_id)
+
+        video_info = self._parse_json(
+            js_to_json(self._html_search_regex(
+                '(?s)videoObject\s*=\s*({.+?});', webpage, 'video object')),
+            video_id)
+
+        formats = []
+        for video_url in re.findall(r'var\s+videourl\s*=\s*"([^"]+)"', webpage):
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.append({
+                    'url': video_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+                    'ext': 'mp4',
+                    'format_id': 'hls',
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': ext,
+                })
         self._sort_formats(formats)
 
+        title = remove_end(self._og_search_title(webpage), ' - Video')
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(video_info.get('length'))
+        timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage, 'upload date'))
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'timestamp': timestamp,
         }

From 8a37aa1517ccc474b3e2831b77e48534cb8ed47c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 01:55:59 +0600
Subject: [PATCH 1474/1515] [extractor/generic] Expand ooyala regex (Closes
 #6485)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6df89f814..649c0bce6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1320,7 +1320,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for Ooyala videos
-        mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
+        mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))

From bf94d763ba73e09fd77d25110c7219254b63c786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 02:00:49 +0600
Subject: [PATCH 1475/1515] [extractor/generic] Add test for #6485

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 649c0bce6..469909a51 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -236,6 +236,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
+        {
+            # ooyala video embedded with http://player.ooyala.com/iframe.js
+            'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
+            'info_dict': {
+                'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
+                'ext': 'mp4',
+                'title': '"Steve Jobs: Man in the Machine" trailer',
+                'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # multiple ooyala embeds on SBN network websites
         {
             'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',

From e0ac521438218e978b9c4bbcd92cfc2d5fef79cb Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Thu, 6 Aug 2015 22:42:58 +0530
Subject: [PATCH 1476/1515] adding support for axel download manager

---
 youtube_dl/downloader/external.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1d5cc9904..30699934b 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -83,6 +83,16 @@ class CurlFD(ExternalFD):
         return cmd
 
 
+class AxelFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        for key, val in info_dict['http_headers'].items():
+            cmd += ['-H', '%s: %s' % (key, val)]
+        cmd += self._configuration_args()
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']

From 5b0c40da24b5ddb789428de731e02ac8759a363c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 03:36:29 +0600
Subject: [PATCH 1477/1515] [extractor/common] Expand meta regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..507ea5ec0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -636,7 +636,7 @@ class InfoExtractor(object):
     @staticmethod
     def _meta_regex(prop):
         return r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    (?=[^>]+(?:itemprop|name|property|id)=(["\']?)%s\1)
                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 
     def _og_search_property(self, prop, html, name=None, **kargs):

From 3550821fb4ca2f0e47542a7fa16b6543b06df724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 03:38:55 +0600
Subject: [PATCH 1478/1515] [periscope] Add extractor (Closes #5850, closes
 #6459)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/periscope.py | 66 +++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 youtube_dl/extractor/periscope.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 922d9b3d8..bd86a5be2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,6 +432,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .periscope import PeriscopeIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
new file mode 100644
index 000000000..5219e1a75
--- /dev/null
+++ b/youtube_dl/extractor/periscope.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    unescapeHTML,
+)
+
+
+class PeriscopeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
+        'md5': '65b57957972e503fcbbaeed8f4fa04ca',
+        'info_dict': {
+            'id': '56102209',
+            'ext': 'mp4',
+            'title': 'Bec Boop - 🚠✈️🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈️🚠 #BoopScope 🎀💗',
+            'timestamp': 1438978559,
+            'upload_date': '20150807',
+            'uploader': 'Bec Boop',
+            'uploader_id': '1465763',
+        },
+        'skip': 'Expires in 24 hours',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        replay = self._download_json(
+            'https://api.periscope.tv/api/v2/getAccessPublic?token=%s' % video_id, video_id)
+
+        video_url = replay['replay_url']
+
+        webpage = self._download_webpage(url, video_id)
+
+        broadcast_data = self._parse_json(
+            unescapeHTML(self._html_search_meta(
+                'broadcast-data', webpage, 'broadcast data', fatal=True)),
+            video_id)
+
+        broadcast = broadcast_data['broadcast']
+        status = broadcast['status']
+
+        uploader = broadcast.get('user_display_name') or broadcast_data.get('user', {}).get('display_name')
+        uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
+
+        title = '%s - %s' % (uploader, status) if uploader else status
+        timestamp = parse_iso8601(broadcast.get('created_at'))
+
+        thumbnails = [{
+            'url': broadcast[image],
+        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
+
+        return {
+            'id': broadcast.get('id') or video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            'protocol': 'm3u8_native',
+            'title': title,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'thumbnails': thumbnails,
+        }

From 621d6a9516e0f9cd8c45e12904f4d4b7615e7fb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 04:00:52 +0600
Subject: [PATCH 1479/1515] [periscope] Switch to API for broadcast data

---
 youtube_dl/extractor/periscope.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 5219e1a75..11648a511 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -25,21 +25,17 @@ class PeriscopeIE(InfoExtractor):
         'skip': 'Expires in 24 hours',
     }
 
+    def _call_api(self, method, token):
+        return self._download_json(
+            'https://api.periscope.tv/api/v2/%s?token=%s' % (method, token), token)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        replay = self._download_json(
-            'https://api.periscope.tv/api/v2/getAccessPublic?token=%s' % video_id, video_id)
+        token = self._match_id(url)
 
+        replay = self._call_api('getAccessPublic', token)
         video_url = replay['replay_url']
 
-        webpage = self._download_webpage(url, video_id)
-
-        broadcast_data = self._parse_json(
-            unescapeHTML(self._html_search_meta(
-                'broadcast-data', webpage, 'broadcast data', fatal=True)),
-            video_id)
-
+        broadcast_data = self._call_api('getBroadcastPublic', token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
 
@@ -54,7 +50,7 @@ class PeriscopeIE(InfoExtractor):
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
         return {
-            'id': broadcast.get('id') or video_id,
+            'id': broadcast.get('id') or token,
             'url': video_url,
             'ext': 'mp4',
             'protocol': 'm3u8_native',

From 1e83741c9a5d67e8bbe65510d41b558361496fe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:33:53 +0600
Subject: [PATCH 1480/1515] [periscope] Add support for running streams

---
 youtube_dl/extractor/periscope.py | 34 +++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 11648a511..de53b752d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,13 +2,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    unescapeHTML,
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
 )
+from ..utils import parse_iso8601
 
 
 class PeriscopeIE(InfoExtractor):
+    IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
     _TEST = {
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -32,9 +34,6 @@ class PeriscopeIE(InfoExtractor):
     def _real_extract(self, url):
         token = self._match_id(url)
 
-        replay = self._call_api('getAccessPublic', token)
-        video_url = replay['replay_url']
-
         broadcast_data = self._call_api('getBroadcastPublic', token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
@@ -43,20 +42,37 @@ class PeriscopeIE(InfoExtractor):
         uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
 
         title = '%s - %s' % (uploader, status) if uploader else status
+        state = broadcast.get('state').lower()
+        if state == 'running':
+            title = self._live_title(title)
         timestamp = parse_iso8601(broadcast.get('created_at'))
 
         thumbnails = [{
             'url': broadcast[image],
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
+        stream = self._call_api('getAccessPublic', token)
+
+        formats = []
+        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
+            video_url = stream.get(format_id + '_url')
+            if not video_url:
+                continue
+            f = {
+                'url': video_url,
+                'ext': 'flv' if format_id == 'rtmp' else 'mp4',
+            }
+            if format_id != 'rtmp':
+                f['protocol'] = 'm3u8_native' if state == 'ended' else 'm3u8'
+            formats.append(f)
+        self._sort_formats(formats)
+
         return {
             'id': broadcast.get('id') or token,
-            'url': video_url,
-            'ext': 'mp4',
-            'protocol': 'm3u8_native',
             'title': title,
             'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'thumbnails': thumbnails,
+            'formats': formats,
         }

From 428e4e4a850df81031e8267dddf759da605639e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:37:38 +0600
Subject: [PATCH 1481/1515] [quickscope] Add extractor

---
 youtube_dl/extractor/__init__.py  |  5 ++++-
 youtube_dl/extractor/periscope.py | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd86a5be2..e38e77a27 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,7 +432,10 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .periscope import PeriscopeIE
+from .periscope import (
+    PeriscopeIE,
+    QuickscopeIE,
+)
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index de53b752d..578b53a24 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -76,3 +76,24 @@ class PeriscopeIE(InfoExtractor):
             'thumbnails': thumbnails,
             'formats': formats,
         }
+
+
+class QuickscopeIE(InfoExtractor):
+    IE_DESC = 'Quisck Scope'
+    _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://watchonperiscope.com/broadcast/56180087',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        broadcast_id = self._match_id(url)
+        request = compat_urllib_request.Request(
+            'https://watchonperiscope.com/api/accessChannel', compat_urllib_parse.urlencode({
+                'broadcast_id': broadcast_id,
+                'entry_ticket': '',
+                'from_push': 'false',
+                'uses_sessions': 'true',
+            }).encode('utf-8'))
+        return self.url_result(
+            self._download_json(request, broadcast_id)['share_url'], 'Periscope')

From b2f82948ee5eadc483c01dc589b82426bb32ba68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:40:41 +0600
Subject: [PATCH 1482/1515] [quickscope] Fix typo

---
 youtube_dl/extractor/periscope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 578b53a24..8ad936758 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -79,7 +79,7 @@ class PeriscopeIE(InfoExtractor):
 
 
 class QuickscopeIE(InfoExtractor):
-    IE_DESC = 'Quisck Scope'
+    IE_DESC = 'Quick Scope'
     _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
     _TEST = {
         'url': 'https://watchonperiscope.com/broadcast/56180087',

From 154655a85ae8b7740aa9fe7821544050fd65641b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 19:21:05 +0600
Subject: [PATCH 1483/1515] [downloader/external] Respect
 --no-check-certificate for wget

---
 youtube_dl/downloader/external.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 30699934b..07ce59f7d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,9 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _no_check_certificate(self, command_option):
+        return [command_option] if self.params.get('nocheckcertificate', False) else []
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -99,6 +102,7 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._no_check_certificate('--no-check-certificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From b465083f45e63fe8aeb0255b5cea7dfbf0770a2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:27:10 +0600
Subject: [PATCH 1484/1515] [sexykarma] Fix test

---
 youtube_dl/extractor/sexykarma.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index 6446d26dc..e33483674 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -29,6 +29,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',

From b61b7787cbef408154695bbb9f5c3d29a70fdd38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:30:57 +0600
Subject: [PATCH 1485/1515] [91porn] Extract age limit

---
 youtube_dl/extractor/porn91.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 72d1b2718..3e15533e9 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -22,6 +22,7 @@ class Porn91IE(InfoExtractor):
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
             'ext': 'mp4',
             'duration': 431,
+            'age_limit': 18,
         }
     }
 
@@ -68,4 +69,5 @@ class Porn91IE(InfoExtractor):
             'url': video_url,
             'duration': duration,
             'comment_count': comment_count,
+            'age_limit': self._rta_search(webpage),
         }

From 8e2b1be12791b4e62c463562b570661e7b2c5852 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:42:50 +0600
Subject: [PATCH 1486/1515] [test/helper] Make age_limit checkable field

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index c8b34654d..cb6eec8d9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -160,7 +160,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                           for key, value in got_dict.items()
-                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location', 'age_limit'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):

From 18c3281f9e1e32e00c778b149137fc91accb3b1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:43:20 +0600
Subject: [PATCH 1487/1515] [24video] Fix test

---
 youtube_dl/extractor/fourtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index b2284ab01..3bb4f6239 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -32,6 +32,7 @@ class FourTubeIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }
 

From 464e792496665b2e3dcabf5df43a45604673730a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:51:21 +0600
Subject: [PATCH 1488/1515] [vpro] Override npo IE_NAME

---
 youtube_dl/extractor/npo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 0c2d02c10..eb12fb810 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -407,6 +407,7 @@ class NPORadioFragmentIE(InfoExtractor):
 
 
 class VPROIE(NPOIE):
+    IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
 
     _TESTS = [

From d7bb8884afc8651b0ad86046dcd56a5330c98dd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:58:24 +0600
Subject: [PATCH 1489/1515] [break] Add age_limit to test

---
 youtube_dl/extractor/breakcom.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 809287d14..aa08051b1 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -18,6 +18,7 @@ class BreakIE(InfoExtractor):
             'id': '2468056',
             'ext': 'mp4',
             'title': 'When Girls Act Like D-Bags',
+            'age_limit': 13,
         }
     }, {
         'url': 'http://www.break.com/video/ugc/baby-flex-2773063',

From 9f2e7c2f34c48942a2a3e55532dd0d0ef8ed4d98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:04:48 +0600
Subject: [PATCH 1490/1515] [ok] Add age_limit to tests

---
 youtube_dl/extractor/odnoklassniki.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 215ffe87b..e5fd1ba04 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -25,6 +25,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
+            'age_limit': 0,
         },
     }, {
         # metadataUrl
@@ -38,6 +39,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': '534380003155',
             'uploader': 'Андрей Мещанинов',
             'like_count': int,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',

From 887e9bc7b561f9b2b97dec8f99f9c04392d95d40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:08:54 +0600
Subject: [PATCH 1491/1515] [ok] Update tests

---
 youtube_dl/extractor/odnoklassniki.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index e5fd1ba04..003d27de7 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -16,12 +16,13 @@ class OdnoklassnikiIE(InfoExtractor):
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
-        'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
+        'md5': '6ba728d85d60aa2e6dd37c9e70fdc6bc',
         'info_dict': {
             'id': '20079905452',
             'ext': 'mp4',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'duration': 100,
+            'upload_date': '20141207',
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
@@ -36,8 +37,9 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Девушка без комплексов ...',
             'duration': 191,
+            'upload_date': '20150518',
             'uploader_id': '534380003155',
-            'uploader': 'Андрей Мещанинов',
+            'uploader': '☭ Андрей Мещанинов ☭',
             'like_count': int,
             'age_limit': 0,
         },

From c8d1be772daa496759bd85cb95c4ec799294c7f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:11:06 +0600
Subject: [PATCH 1492/1515] [rutube] Add age_limit to test

---
 youtube_dl/extractor/rutube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 5b1c3577a..d94dc7399 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -30,6 +30,7 @@ class RutubeIE(InfoExtractor):
             'uploader': 'NTDRussian',
             'uploader_id': '29790',
             'upload_date': '20131016',
+            'age_limit': 0,
         },
         'params': {
             # It requires ffmpeg (m3u8 download)

From 08df685fe7764ef9f7dc271075340e4effc5e621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 08:51:37 +0600
Subject: [PATCH 1493/1515] [videolectures] Fix _VALID_URL for
 test_no_duplicates to pass

---
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 24584dc80..ef2da5632 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)(?:/?[#?].*)?$'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
     _TEST = {

From 12bb392a0ff8adbde2ced75b0c4976d0aabc7f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 17:10:40 +0600
Subject: [PATCH 1494/1515] [vimeo] Fix password protected videos (Closes
 #6507)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 10d6745af..4c4e3c72a 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -203,7 +203,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'xsrft=%s' % token)
+        password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')

From 8d6765cf48138cc44fdbaee4e8c7a199ae348bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:07:18 +0600
Subject: [PATCH 1495/1515] [extractor/generic] Add generic support for xspf
 playist extraction

---
 youtube_dl/extractor/common.py | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index def6caa0d..e201ea6db 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -39,6 +39,8 @@ from ..utils import (
     sanitize_filename,
     unescapeHTML,
     url_basename,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
@@ -1142,6 +1144,45 @@ class InfoExtractor(object):
             })
         return subtitles
 
+    def _extract_xspf_playlist(self, playlist_url, playlist_id):
+        playlist = self._download_xml(
+            playlist_url, playlist_id, 'Downloading xpsf playlist',
+            'Unable to download xspf manifest')
+
+        NS_MAP = {
+            'xspf': 'http://xspf.org/ns/0/',
+            's1': 'http://static.streamone.nl/player/ns/0',
+        }
+
+        entries = []
+        for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
+            title = xpath_text(
+                track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+            description = xpath_text(
+                track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+            thumbnail = xpath_text(
+                track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+            duration = float_or_none(
+                xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
+
+            formats = [{
+                'url': location.text,
+                'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+                'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+                'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+            } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
+            self._sort_formats(formats)
+
+            entries.append({
+                'id': playlist_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+            })
+        return entries
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From e0b9d78fab76e2c2819c8a9a7512ad4533319b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:09:50 +0600
Subject: [PATCH 1496/1515] [extractor/common] Clarify playlists can have
 description field

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e201ea6db..9b4775e0a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -204,8 +204,8 @@ class InfoExtractor(object):
     There must be a key "entries", which is a list, an iterable, or a PagedList
     object, each element of which is a valid dictionary by this specification.
 
-    Additionally, playlists can have "title" and "id" attributes with the same
-    semantics as videos (see above).
+    Additionally, playlists can have "title", "description" and "id" attributes
+    with the same semantics as videos (see above).
 
 
     _type "multi_video" indicates that there are multiple videos that

From 3a30508b943c044e5f684b703ff58ac352686f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:11:23 +0600
Subject: [PATCH 1497/1515] [telegraaf] Add extractor (Closes #6492)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/telegraaf.py | 35 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/telegraaf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e38e77a27..dad3ec87f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -596,6 +596,7 @@ from .techtalks import TechTalksIE
 from .ted import TEDIE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
+from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
new file mode 100644
index 000000000..6f8333cfc
--- /dev/null
+++ b/youtube_dl/extractor/telegraaf.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class TelegraafIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?telegraaf\.nl/tv/(?:[^/]+/)+(?P<id>\d+)/[^/]+\.html'
+    _TEST = {
+        'url': 'http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html',
+        'md5': '83245a9779bcc4a24454bfd53c65b6dc',
+        'info_dict': {
+            'id': '24353229',
+            'ext': 'mp4',
+            'title': 'Tikibad ontruimd wegens brand',
+            'description': 'md5:05ca046ff47b931f9b04855015e163a4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 33,
+        },
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist_url = self._search_regex(
+            r"iframe\.loadPlayer\('([^']+)'", webpage, 'player')
+
+        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
+        title = remove_end(self._og_search_title(webpage), ' - VIDEO')
+        description = self._og_search_description(webpage)
+
+        return self.playlist_result(entries, playlist_id, title, description)

From f32143469fd0a2720bd40908ea8360490983b97d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:15:00 +0600
Subject: [PATCH 1498/1515] [tweakers] Use _extract_xspf_playlist

---
 youtube_dl/extractor/tweakers.py | 42 +++-----------------------------
 1 file changed, 4 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index c80ec15cf..4bbe76e96 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,41 +25,7 @@ class TweakersIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        playlist = self._download_xml(
-            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
-            video_id)
-
-        NS_MAP = {
-            'xspf': 'http://xspf.org/ns/0/',
-            's1': 'http://static.streamone.nl/player/ns/0',
-        }
-
-        track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
-
-        title = xpath_text(
-            track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
-        description = xpath_text(
-            track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
-        thumbnail = xpath_text(
-            track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
-        duration = float_or_none(
-            xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
-            1000)
-
-        formats = [{
-            'url': location.text,
-            'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
-            'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
-            'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
-        } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        playlist_id = self._match_id(url)
+        entries = self._extract_xspf_playlist(
+            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % playlist_id, playlist_id)
+        return self.playlist_result(entries, playlist_id)

From 0dcb318f622d944ad0f5c23c32c9bc9b00e76aaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:15:20 +0600
Subject: [PATCH 1499/1515] [tweakers] Fix test

---
 youtube_dl/extractor/tweakers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 4bbe76e96..6eeffb1cc 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -13,7 +13,7 @@ class TweakersIE(InfoExtractor):
     _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': '1b5afa817403bb5baa08359dca31e6df',
+        'md5': '3147e4ddad366f97476a93863e4557c8',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',

From 98044462b1035000a44b35a41f4f780b2e844f2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:18:50 +0600
Subject: [PATCH 1500/1515] [extractor/common] Use playlist id as default title

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9b4775e0a..be91e03e9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1157,7 +1157,7 @@ class InfoExtractor(object):
         entries = []
         for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
             title = xpath_text(
-                track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+                track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
             description = xpath_text(
                 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
             thumbnail = xpath_text(

From fb2f339fec20c35cb62c1da682e0dfd418faef81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:21:25 +0600
Subject: [PATCH 1501/1515] [dhm] Use _extract_xspf_playlist

---
 youtube_dl/extractor/dhm.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 3ed1f1663..127eb0439 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -34,24 +34,14 @@ class DHMIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        playlist_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, playlist_id)
 
         playlist_url = self._search_regex(
             r"file\s*:\s*'([^']+)'", webpage, 'playlist url')
 
-        playlist = self._download_xml(playlist_url, video_id)
-
-        track = playlist.find(
-            './{http://xspf.org/ns/0/}trackList/{http://xspf.org/ns/0/}track')
-
-        video_url = xpath_text(
-            track, './{http://xspf.org/ns/0/}location',
-            'video url', fatal=True)
-        thumbnail = xpath_text(
-            track, './{http://xspf.org/ns/0/}image',
-            'thumbnail')
+        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
 
         title = self._search_regex(
             [r'dc:title="([^"]+)"', r'<title> &raquo;([^<]+)</title>'],
@@ -63,11 +53,10 @@ class DHMIE(InfoExtractor):
             r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
             webpage, 'duration', default=None))
 
-        return {
-            'id': video_id,
-            'url': video_url,
+        entries[0].update({
             'title': title,
             'description': description,
             'duration': duration,
-            'thumbnail': thumbnail,
-        }
+        })
+
+        return self.playlist_result(entries, playlist_id)

From 942acef594428b5f5c7e0ed7860cb6d725d8f1e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:41:55 +0600
Subject: [PATCH 1502/1515] [extractor/common] Extract _parse_xspf

---
 youtube_dl/extractor/common.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index be91e03e9..5982055be 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1144,11 +1144,15 @@ class InfoExtractor(object):
             })
         return subtitles
 
-    def _extract_xspf_playlist(self, playlist_url, playlist_id):
-        playlist = self._download_xml(
+    def _extract_xspf_playlist(self, playlist_url, playlist_id, fatal=True):
+        xspf = self._download_xml(
             playlist_url, playlist_id, 'Downloading xpsf playlist',
-            'Unable to download xspf manifest')
+            'Unable to download xspf manifest', fatal=fatal)
+        if xspf is False:
+            return []
+        return self._parse_xspf(xspf, playlist_id)
 
+    def _parse_xspf(self, playlist, playlist_id):
         NS_MAP = {
             'xspf': 'http://xspf.org/ns/0/',
             's1': 'http://static.streamone.nl/player/ns/0',

From 729accb48221bd72e40076939616792c1c6fc15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:43:42 +0600
Subject: [PATCH 1503/1515] [extractor/generic] Add support for xspf playlists

---
 youtube_dl/extractor/generic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 901f77304..a382d6be4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1198,6 +1198,8 @@ class GenericIE(InfoExtractor):
                 return self._extract_rss(url, video_id, doc)
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
                 return self._parse_smil(doc, url, video_id)
+            elif doc.tag == '{http://xspf.org/ns/0/}playlist':
+                return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
         except compat_xml_parse_error:
             pass
 
@@ -1799,7 +1801,8 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
-            if determine_ext(video_url) == 'smil':
+            ext = determine_ext(video_url)
+            if ext == 'smil':
                 entries.append({
                     'id': video_id,
                     'formats': self._extract_smil_formats(video_url, video_id),
@@ -1807,6 +1810,8 @@ class GenericIE(InfoExtractor):
                     'title': video_title,
                     'age_limit': age_limit,
                 })
+            elif ext == 'xspf':
+                return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             else:
                 entries.append({
                     'id': video_id,

From 1de5cd3ba51ce67d9a1cd3b40157058e78e46692 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:47:08 +0600
Subject: [PATCH 1504/1515] [extractor/generic] Add test for xspf playlist

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a382d6be4..4756a658f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -198,6 +198,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
+        {
+            'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
+            'info_dict': {
+                'id': 'mZlp2ctYIUEB',
+                'ext': 'mp4',
+                'title': 'Tikibad ontruimd wegens brand',
+                'description': 'md5:05ca046ff47b931f9b04855015e163a4',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 33,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 0791ac1b4415601f464f9656a4485b3ae6b67f4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:47:58 +0600
Subject: [PATCH 1505/1515] [extractor/generic] Clarify comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4756a658f..376feecae 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1206,7 +1206,7 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed or a SMIL file?
+        # Is it an RSS feed, a SMIL file or a XSPF playlist?
         try:
             doc = parse_xml(webpage)
             if doc.tag == 'rss':

From 27c7114af6b82bfe8be6b8e4dfa6e11dd1356044 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 9 Aug 2015 20:13:02 +0200
Subject: [PATCH 1506/1515] release 2015.08.09

---
 README.md              | 2 +-
 docs/supportedsites.md | 8 ++++++--
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 02b9775f9..15baf75ce 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,httpie,wget
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,axel,curl,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 657935dc6..e21471102 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -86,7 +86,7 @@
  - **chirbit:profile**
  - **Cinchcast**
  - **Cinemassacre**
- - **clipfish**
+ - **Clipfish**
  - **cliphunter**
  - **Clipsyndicate**
  - **Cloudy**
@@ -116,6 +116,7 @@
  - **DailymotionCloud**
  - **daum.net**
  - **DBTV**
+ - **DCN**
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
@@ -351,7 +352,6 @@
  - **NowTV**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
- - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
@@ -377,6 +377,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
+ - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
@@ -406,6 +407,7 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
+ - **Quickscope**: Quick Scope
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -518,6 +520,7 @@
  - **ted**
  - **TeleBruxelles**
  - **telecinco.es**
+ - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
  - **TenPlay**
@@ -621,6 +624,7 @@
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
+ - **vpro**: npo.nl and ntr.nl
  - **VRT**
  - **vube**: Vube.com
  - **VuClip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9f209499c..6462d4477 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.06.1'
+__version__ = '2015.08.09'

From c5864a8ce6379dca300f447cca12a5a946d67d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 21:38:58 +0600
Subject: [PATCH 1507/1515] [fc2] Fix python 2.6 (Closes #6512)

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 1ccc1a964..e4f7195a8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -86,7 +86,7 @@ class FC2IE(InfoExtractor):
 
         info_url = (
             "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&".
-            format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.', '%2E')))
+            format(video_id, mimi, compat_urllib_request.quote(refer, safe=b'').replace('.', '%2E')))
 
         info_webpage = self._download_webpage(
             info_url, video_id, note='Downloading info page')

From f6c3664d717857a7994f189a01a00402df2b4168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 23:35:08 +0600
Subject: [PATCH 1508/1515] [vimeo] Fix login (Closes #6488)

---
 youtube_dl/extractor/vimeo.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4c4e3c72a..5bce78ac0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -29,6 +29,7 @@ from ..utils import (
 class VimeoBaseInfoExtractor(InfoExtractor):
     _NETRC_MACHINE = 'vimeo'
     _LOGIN_REQUIRED = False
+    _LOGIN_URL = 'https://vimeo.com/log_in'
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -37,21 +38,25 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
             return
         self.report_login()
-        login_url = 'https://vimeo.com/log_in'
-        webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        webpage = self._download_webpage(self._LOGIN_URL, None, False)
+        token = self._extract_xsrft(webpage)
         data = urlencode_postdata({
+            'action': 'login',
             'email': username,
             'password': password,
-            'action': 'login',
             'service': 'vimeo',
             'token': token,
         })
-        login_request = compat_urllib_request.Request(login_url, data)
+        login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        login_request.add_header('Cookie', 'xsrft=%s' % token)
+        login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
+    def _extract_xsrft(self, webpage):
+        return self._search_regex(
+            r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
+            webpage, 'login token', group='xsrft')
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -193,7 +198,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
+        token = self._extract_xsrft(webpage)
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -422,7 +427,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         }
 
 
-class VimeoChannelIE(InfoExtractor):
+class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
@@ -453,7 +458,7 @@ class VimeoChannelIE(InfoExtractor):
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
-        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
+        token = self._extract_xsrft(webpage)
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)
@@ -603,7 +608,7 @@ class VimeoReviewIE(InfoExtractor):
         return self.url_result(player_url, 'Vimeo', video_id)
 
 
-class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
+class VimeoWatchLaterIE(VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
     _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'

From 844587669eeffaf139b9c8888550c9d3b88335e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 23:58:01 +0600
Subject: [PATCH 1509/1515] [vimeo:watchlater] Fix extraction (Closes #3886)

---
 youtube_dl/extractor/vimeo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5bce78ac0..1eeb4618e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -431,6 +431,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
+    _TITLE = None
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
     _TESTS = [{
         'url': 'https://vimeo.com/channels/tributes',
@@ -445,7 +446,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         return '%s/videos/page:%d/' % (base_url, pagenum)
 
     def _extract_list_title(self, webpage):
-        return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
+        return self._TITLE or self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
     def _login_list_password(self, page_url, list_id, webpage):
         login_form = self._search_regex(
@@ -611,11 +612,11 @@ class VimeoReviewIE(InfoExtractor):
 class VimeoWatchLaterIE(VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
-    _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'
+    _VALID_URL = r'https://vimeo\.com/(?:home/)?watchlater|:vimeowatchlater'
+    _TITLE = 'Watch Later'
     _LOGIN_REQUIRED = True
-    _TITLE_RE = r'href="/home/watchlater".*?>(.*?)<'
     _TESTS = [{
-        'url': 'https://vimeo.com/home/watchlater',
+        'url': 'https://vimeo.com/watchlater',
         'only_matching': True,
     }]
 
@@ -631,7 +632,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
         return request
 
     def _real_extract(self, url):
-        return self._extract_videos('watchlater', 'https://vimeo.com/home/watchlater')
+        return self._extract_videos('watchlater', 'https://vimeo.com/watchlater')
 
 
 class VimeoLikesIE(InfoExtractor):

From 11b5605815d685263b271b4e061c43f9cb55a08c Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puckipedia.nl>
Date: Mon, 10 Aug 2015 20:52:38 +0200
Subject: [PATCH 1510/1515] [youtube] Use the first v= argument in the URL

This is according to how youtube handles multiple v= values in one URL.
Before this, it was possible to make a single URL show up differently
 on youtube itself, and if you downloaded/viewed it with youtube-dl/mpv
---
 youtube_dl/extractor/youtube.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 67a1df9a0..eaf058cfb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -213,7 +213,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
-                                 (?:.*?&)?                                    # any other preceding param (like /?s=tuff&v=xxxx)
+                                 (?:.*?&)??                                   # any other preceding param (like /?s=tuff&v=xxxx)
                                  v=
                              )
                          ))
@@ -380,6 +380,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'setindia'
             }
         },
+        {
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
+            'note': 'Use the first video ID in the URL',
+            'info_dict': {
+                'id': 'BaW_jenozKc',
+                'ext': 'mp4',
+                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+                'uploader': 'Philipp Hagemeister',
+                'uploader_id': 'phihag',
+                'upload_date': '20121002',
+                'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
+                'categories': ['Science & Technology'],
+                'tags': ['youtube-dl'],
+                'like_count': int,
+                'dislike_count': int,
+            }
+        },
         {
             'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
             'note': '256k DASH audio (format 141) via DASH manifest',

From b29440aee64027b3e4145070b0235193752b4d9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:17:41 +0600
Subject: [PATCH 1511/1515] [vimeo:user] Do not match watchlater

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1eeb4618e..50df79ca1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -505,7 +505,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
 
 class VimeoUserIE(VimeoChannelIE):
     IE_NAME = 'vimeo:user'
-    _VALID_URL = r'https://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
+    _VALID_URL = r'https://vimeo\.com/(?!(?:[0-9]+|watchlater)(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
     _TESTS = [{
         'url': 'https://vimeo.com/nkistudio/videos',

From 34a7de2970d8bbceeb3f485d64a57f67489a44d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:22:06 +0600
Subject: [PATCH 1512/1515] [youtube] Skip download for multiple v= test

---
 youtube_dl/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eaf058cfb..01dbbfa3c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -395,7 +395,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'tags': ['youtube-dl'],
                 'like_count': int,
                 'dislike_count': int,
-            }
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',

From 34952f09e175e0b78c929fddf56f82ccf028dc5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:24:53 +0600
Subject: [PATCH 1513/1515] [youtube] Add age limit to tests

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 01dbbfa3c..e74a39095 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -365,6 +365,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'age_limit': 18,
             }
         },
         {
@@ -475,6 +476,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',
+                'age_limit': 18,
             },
         },
         # Age-gate video with encrypted signature
@@ -488,6 +490,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
                 'upload_date': '20110629',
+                'age_limit': 18,
             },
         },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)

From fb0d12c6cbcabd6f9e84d51c82dea6778d0bb863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:46:25 +0600
Subject: [PATCH 1514/1515] [pbs] Add age limit to tests

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index a53479aad..683c81de3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -92,6 +92,7 @@ class PBSIE(InfoExtractor):
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
+                'age_limit': 10,
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg

From b1ac38fadc65049dc6f9611fa7e9649de1e7eb93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:49:23 +0600
Subject: [PATCH 1515/1515] [tvplay] Add age limit to tests

---
 youtube_dl/extractor/tvplay.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 79863e781..b4683de54 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -104,6 +104,7 @@ class TVPlayIE(InfoExtractor):
                 'duration': 1492,
                 'timestamp': 1330522854,
                 'upload_date': '20120229',
+                'age_limit': 18,
             },
             'params': {
                 # rtmp download